23 from builtins
import str
29 import lsst.daf.persistence
as dafPersist
30 from .
import ImageMapping, ExposureMapping, CalibrationMapping, DatasetMapping
31 import lsst.afw.geom
as afwGeom
32 import lsst.afw.image
as afwImage
33 import lsst.afw.table
as afwTable
34 import lsst.afw.cameraGeom
as afwCameraGeom
35 import lsst.log
as lsstLog
36 import lsst.pex.policy
as pexPolicy
37 import lsst.pex.exceptions
as pexExcept
38 from .exposureIdInfo
import ExposureIdInfo
39 from .makeRawVisitInfo
import MakeRawVisitInfo
40 from lsst.utils
import getPackageDir
42 """This module defines the CameraMapper base class."""
47 """CameraMapper is a base class for mappers that handle images from a
48 camera and products derived from them. This provides an abstraction layer
49 between the data on disk and the code.
51 Public methods: keys, queryMetadata, getDatasetTypes, map,
52 canStandardize, standardize
54 Mappers for specific data sources (e.g., CFHT Megacam, LSST
55 simulations, etc.) should inherit this class.
57 The CameraMapper manages datasets within a "root" directory. Note that
58 writing to a dataset present in the input root will hide the existing
59 dataset but not overwrite it. See #2160 for design discussion.
61 A camera is assumed to consist of one or more rafts, each composed of
62 multiple CCDs. Each CCD is in turn composed of one or more amplifiers
63 (amps). A camera is also assumed to have a camera geometry description
64 (CameraGeom object) as a policy file, a filter description (Filter class
65 static configuration) as another policy file, and an optional defects
66 description directory.
68 Information from the camera geometry and defects are inserted into all
69 Exposure objects returned.
71 The mapper uses one or two registries to retrieve metadata about the
72 images. The first is a registry of all raw exposures. This must contain
73 the time of the observation. One or more tables (or the equivalent)
74 within the registry are used to look up data identifier components that
75 are not specified by the user (e.g. filter) and to return results for
76 metadata queries. The second is an optional registry of all calibration
77 data. This should contain validity start and end entries for each
78 calibration dataset in the same timescale as the observation time.
80 Subclasses will typically set MakeRawVisitInfoClass:
82 MakeRawVisitInfoClass: a class variable that points to a subclass of
83 MakeRawVisitInfo, a functor that creates an
84 lsst.afw.image.VisitInfo from the FITS metadata of a raw image.
86 Subclasses must provide the following methods:
88 _extractDetectorName(self, dataId): returns the detector name for a CCD
89 (e.g., "CFHT 21", "R:1,2 S:3,4") as used in the AFW CameraGeom class given
90 a dataset identifier referring to that CCD or a subcomponent of it.
92 _computeCcdExposureId(self, dataId): see below
94 _computeCoaddExposureId(self, dataId, singleFilter): see below
96 Subclasses may also need to override the following methods:
98 _transformId(self, dataId): transformation of a data identifier
99 from colloquial usage (e.g., "ccdname") to proper/actual usage (e.g., "ccd"),
100 including making suitable for path expansion (e.g. removing commas).
101 The default implementation does nothing. Note that this
102 method should not modify its input parameter.
104 getShortCcdName(self, ccdName): a static method that returns a shortened name
105 suitable for use as a filename. The default version converts spaces to underscores.
107 _getCcdKeyVal(self, dataId): return a CCD key and value
108 by which to look up defects in the defects registry.
109 The default value returns ("ccd", detector name)
111 _mapActualToPath(self, template, actualId): convert a template path to an
112 actual path, using the actual dataset identifier.
114 The mapper's behaviors are largely specified by the policy file.
115 See the MapperDictionary.paf for descriptions of the available items.
117 The 'exposures', 'calibrations', and 'datasets' subpolicies configure
118 mappings (see Mappings class).
120 Common default mappings for all subclasses can be specified in the
121 "policy/{images,exposures,calibrations,datasets}.yaml" files. This provides
122 a simple way to add a product to all camera mappers.
124 Functions to map (provide a path to the data given a dataset
125 identifier dictionary) and standardize (convert data into some standard
126 format or type) may be provided in the subclass as "map_{dataset type}"
127 and "std_{dataset type}", respectively.
129 If non-Exposure datasets cannot be retrieved using standard
130 daf_persistence methods alone, a "bypass_{dataset type}" function may be
131 provided in the subclass to return the dataset instead of using the
132 "datasets" subpolicy.
134 Implementations of map_camera and bypass_camera that should typically be
135 sufficient are provided in this base class.
138 * Handle defects the same was as all other calibration products, using the calibration registry
139 * Instead of auto-loading the camera at construction time, load it from the calibration registry
140 * Rewrite defects as AFW tables so we don't need pyfits to unpersist them; then remove all mention
141 of pyfits from this package.
147 MakeRawVisitInfoClass = MakeRawVisitInfo
150 PupilFactoryClass = afwCameraGeom.PupilFactory
152 def __init__(self, policy, repositoryDir,
153 root=
None, registry=
None, calibRoot=
None, calibRegistry=
None,
154 provided=
None, parentRegistry=
None, repositoryCfg=
None):
155 """Initialize the CameraMapper.
159 policy : daf_persistence.Policy,
160 Can also be pexPolicy.Policy, only for backward compatibility.
161 Policy with per-camera defaults already merged.
162 repositoryDir : string
163 Policy repository for the subclassing module (obtained with
164 getRepositoryPath() on the per-camera default dictionary).
165 root : string, optional
166 Path to the root directory for data.
167 registry : string, optional
168 Path to registry with data's metadata.
169 calibRoot : string, optional
170 Root directory for calibrations.
171 calibRegistry : string, optional
172 Path to registry with calibrations' metadata.
173 provided : list of string, optional
174 Keys provided by the mapper.
175 parentRegistry : Registry subclass, optional
176 Registry from a parent repository that may be used to look up
178 repositoryCfg : daf_persistence.RepositoryCfg or None, optional
179 The configuration information for the repository this mapper is
183 dafPersist.Mapper.__init__(self)
185 self.
log = lsstLog.Log.getLogger(
"CameraMapper")
190 self.
root = repositoryCfg.root
193 if isinstance(policy, pexPolicy.Policy):
194 policy = dafPersist.Policy(policy)
196 repoPolicy = repositoryCfg.policy
if repositoryCfg
else None
197 if repoPolicy
is not None:
198 policy.update(repoPolicy)
200 defaultPolicyFile = dafPersist.Policy.defaultPolicyFile(
"obs_base",
201 "MapperDictionary.paf",
203 dictPolicy = dafPersist.Policy(defaultPolicyFile)
204 policy.merge(dictPolicy)
208 if 'levels' in policy:
209 levelsPolicy = policy[
'levels']
210 for key
in levelsPolicy.names(
True):
211 self.
levels[key] = set(levelsPolicy.asArray(key))
214 if 'defaultSubLevels' in policy:
220 root = dafPersist.LogicalLocation(root).locString()
230 if calibRoot
is not None:
231 calibRoot = dafPersist.Storage.absolutePath(root, calibRoot)
232 calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
235 calibRoot = policy.get(
'calibRoot',
None)
237 calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
239 if calibStorage
is None:
247 posixIfNoSql=(
not parentRegistry))
250 needCalibRegistry = policy.get(
'needCalibRegistry',
None)
251 if needCalibRegistry:
254 "calibRegistryPath", calibStorage)
257 "'needCalibRegistry' is true in Policy, but was unable to locate a repo at " +
258 "calibRoot ivar:%s or policy['calibRoot']:%s" %
259 (calibRoot, policy.get(
'calibRoot',
None)))
275 if 'defects' in policy:
276 self.
defectPath = os.path.join(repositoryDir, policy[
'defects'])
277 defectRegistryLocation = os.path.join(self.
defectPath,
"defectRegistry.sqlite3")
278 self.
defectRegistry = dafPersist.Registry.create(defectRegistryLocation)
289 raise ValueError(
'class variable packageName must not be None')
293 def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None):
294 """Initialize mappings
296 For each of the dataset types that we want to be able to read, there are
297 methods that can be created to support them:
298 * map_<dataset> : determine the path for dataset
299 * std_<dataset> : standardize the retrieved dataset
300 * bypass_<dataset> : retrieve the dataset (bypassing the usual retrieval machinery)
301 * query_<dataset> : query the registry
303 Besides the dataset types explicitly listed in the policy, we create
304 additional, derived datasets for additional conveniences, e.g., reading
305 the header of an image, retrieving only the size of a catalog.
307 @param policy (Policy) Policy with per-camera defaults already merged
308 @param rootStorage (Storage subclass instance) Interface to persisted repository data
309 @param calibRoot (Storage subclass instance) Interface to persisted calib repository data
310 @param provided (list of strings) Keys provided by the mapper
313 imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
314 "obs_base",
"ImageMappingDictionary.paf",
"policy"))
315 expMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
316 "obs_base",
"ExposureMappingDictionary.paf",
"policy"))
317 calMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
318 "obs_base",
"CalibrationMappingDictionary.paf",
"policy"))
319 dsMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
320 "obs_base",
"DatasetMappingDictionary.paf",
"policy"))
324 (
"images", imgMappingPolicy, ImageMapping),
325 (
"exposures", expMappingPolicy, ExposureMapping),
326 (
"calibrations", calMappingPolicy, CalibrationMapping),
327 (
"datasets", dsMappingPolicy, DatasetMapping)
330 for name, defPolicy, cls
in mappingList:
332 datasets = policy[name]
335 defaultsPath = os.path.join(getPackageDir(
"obs_base"),
"policy", name +
".yaml")
336 if os.path.exists(defaultsPath):
337 datasets.merge(dafPersist.Policy(defaultsPath))
340 setattr(self, name, mappings)
341 for datasetType
in datasets.names(
True):
342 subPolicy = datasets[datasetType]
343 subPolicy.merge(defPolicy)
345 if not hasattr(self,
"map_" + datasetType)
and 'composite' in subPolicy:
346 def compositeClosure(dataId, write=False, mapper=None, mapping=None,
347 subPolicy=subPolicy):
348 components = subPolicy.get(
'composite')
349 assembler = subPolicy[
'assembler']
if 'assembler' in subPolicy
else None
350 disassembler = subPolicy[
'disassembler']
if 'disassembler' in subPolicy
else None
351 python = subPolicy[
'python']
352 butlerComposite = dafPersist.ButlerComposite(assembler=assembler,
353 disassembler=disassembler,
357 for name, component
in components.items():
358 butlerComposite.add(id=name,
359 datasetType=component.get(
'datasetType'),
360 setter=component.get(
'setter',
None),
361 getter=component.get(
'getter',
None),
362 subset=component.get(
'subset',
False),
363 inputOnly=component.get(
'inputOnly',
False))
364 return butlerComposite
365 setattr(self,
"map_" + datasetType, compositeClosure)
369 if name ==
"calibrations":
371 provided=provided, dataRoot=rootStorage)
373 mapping = cls(datasetType, subPolicy, self.
registry, rootStorage, provided=provided)
374 self.keyDict.update(mapping.keys())
375 mappings[datasetType] = mapping
376 self.
mappings[datasetType] = mapping
377 if not hasattr(self,
"map_" + datasetType):
378 def mapClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
379 return mapping.map(mapper, dataId, write)
380 setattr(self,
"map_" + datasetType, mapClosure)
381 if not hasattr(self,
"query_" + datasetType):
382 def queryClosure(format, dataId, mapping=mapping):
383 return mapping.lookup(format, dataId)
384 setattr(self,
"query_" + datasetType, queryClosure)
385 if hasattr(mapping,
"standardize")
and not hasattr(self,
"std_" + datasetType):
386 def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
387 return mapping.standardize(mapper, item, dataId)
388 setattr(self,
"std_" + datasetType, stdClosure)
390 def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
391 """Set convenience methods on CameraMapper"""
392 mapName =
"map_" + datasetType +
"_" + suffix
393 bypassName =
"bypass_" + datasetType +
"_" + suffix
394 queryName =
"query_" + datasetType +
"_" + suffix
395 if not hasattr(self, mapName):
396 setattr(self, mapName, mapImpl
or getattr(self,
"map_" + datasetType))
397 if not hasattr(self, bypassName):
398 if bypassImpl
is None and hasattr(self,
"bypass_" + datasetType):
399 bypassImpl = getattr(self,
"bypass_" + datasetType)
400 if bypassImpl
is not None:
401 setattr(self, bypassName, bypassImpl)
402 if not hasattr(self, queryName):
403 setattr(self, queryName, queryImpl
or getattr(self,
"query_" + datasetType))
406 setMethods(
"filename", bypassImpl=
lambda datasetType, pythonType, location, dataId:
407 [os.path.join(location.getStorage().root, p)
for p
in location.getLocations()])
409 if subPolicy[
"storage"] ==
"FitsStorage":
410 setMethods(
"md", bypassImpl=
lambda datasetType, pythonType, location, dataId:
411 afwImage.readMetadata(location.getLocationsWithRoot()[0]))
412 if name ==
"exposures":
413 setMethods(
"wcs", bypassImpl=
lambda datasetType, pythonType, location, dataId:
415 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
416 setMethods(
"calib", bypassImpl=
lambda datasetType, pythonType, location, dataId:
418 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
419 setMethods(
"visitInfo",
420 bypassImpl=
lambda datasetType, pythonType, location, dataId:
422 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
424 bypassImpl=
lambda datasetType, pythonType, location, dataId:
426 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
427 setMethods(
"detector",
428 mapImpl=
lambda dataId, write=
False:
429 dafPersist.ButlerLocation(
430 pythonType=
"lsst.afw.cameraGeom.CameraConfig",
432 storageName=
"Internal",
433 locationList=
"ignored",
438 bypassImpl=
lambda datasetType, pythonType, location, dataId:
441 setMethods(
"bbox", bypassImpl=
lambda dsType, pyType, location, dataId:
442 afwImage.bboxFromMetadata(
443 afwImage.readMetadata(location.getLocationsWithRoot()[0], hdu=1)))
445 elif name ==
"images":
446 setMethods(
"bbox", bypassImpl=
lambda dsType, pyType, location, dataId:
447 afwImage.bboxFromMetadata(
448 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
450 if subPolicy[
"storage"] ==
"FitsCatalogStorage":
451 setMethods(
"md", bypassImpl=
lambda datasetType, pythonType, location, dataId:
452 afwImage.readMetadata(os.path.join(location.getStorage().root,
453 location.getLocations()[0]), hdu=1))
456 if subPolicy[
"storage"] ==
"FitsStorage":
457 def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
458 subId = dataId.copy()
460 loc = mapping.map(mapper, subId, write)
461 bbox = dataId[
'bbox']
462 llcX = bbox.getMinX()
463 llcY = bbox.getMinY()
464 width = bbox.getWidth()
465 height = bbox.getHeight()
466 loc.additionalData.set(
'llcX', llcX)
467 loc.additionalData.set(
'llcY', llcY)
468 loc.additionalData.set(
'width', width)
469 loc.additionalData.set(
'height', height)
470 if 'imageOrigin' in dataId:
471 loc.additionalData.set(
'imageOrigin',
472 dataId[
'imageOrigin'])
475 def querySubClosure(key, format, dataId, mapping=mapping):
476 subId = dataId.copy()
478 return mapping.lookup(format, subId)
479 setMethods(
"sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)
481 if subPolicy[
"storage"] ==
"FitsCatalogStorage":
483 setMethods(
"len", bypassImpl=
lambda datasetType, pythonType, location, dataId:
484 afwImage.readMetadata(os.path.join(location.getStorage().root,
485 location.getLocations()[0]),
486 hdu=1).get(
"NAXIS2"))
489 if not datasetType.endswith(
"_schema")
and datasetType +
"_schema" not in datasets:
490 setMethods(
"schema", bypassImpl=
lambda datasetType, pythonType, location, dataId:
491 afwTable.Schema.readFits(os.path.join(location.getStorage().root,
492 location.getLocations()[0])))
494 def _computeCcdExposureId(self, dataId):
495 """Compute the 64-bit (long) identifier for a CCD exposure.
497 Subclasses must override
499 @param dataId (dict) Data identifier with visit, ccd
501 raise NotImplementedError()
503 def _computeCoaddExposureId(self, dataId, singleFilter):
504 """Compute the 64-bit (long) identifier for a coadd.
506 Subclasses must override
508 @param dataId (dict) Data identifier with tract and patch.
509 @param singleFilter (bool) True means the desired ID is for a single-
510 filter coadd, in which case dataId
513 raise NotImplementedError()
515 def _search(self, path):
516 """Search for path in the associated repository's storage.
521 Path that describes an object in the repository associated with
523 Path may contain an HDU indicator, e.g. 'foo.fits[1]'. The
524 indicator will be stripped when searching and so will match
525 filenames without the HDU indicator, e.g. 'foo.fits'. The path
526 returned WILL contain the indicator though, e.g. ['foo.fits[1]'].
531 The path for this object in the repository. Will return None if the
532 object can't be found. If the input argument path contained an HDU
533 indicator, the returned path will also contain the HDU indicator.
535 return self.rootStorage.search(path)
538 """Rename any existing object with the given type and dataId.
540 The CameraMapper implementation saves objects in a sequence of e.g.:
544 All of the backups will be placed in the output repo, however, and will
545 not be removed if they are found elsewhere in the _parent chain. This
546 means that the same file will be stored twice if the previous version was
547 found in an input repo.
556 def firstElement(list):
557 """Get the first element in the list, or None if that can't be done.
559 return list[0]
if list
is not None and len(list)
else None
562 newLocation = self.map(datasetType, dataId, write=
True)
563 newPath = newLocation.getLocations()[0]
564 path = dafPersist.PosixStorage.search(self.
root, newPath, searchParents=
True)
565 path = firstElement(path)
567 while path
is not None:
569 oldPaths.append((n, path))
570 path = dafPersist.PosixStorage.search(self.
root,
"%s~%d" % (newPath, n), searchParents=
True)
571 path = firstElement(path)
572 for n, oldPath
in reversed(oldPaths):
573 self.rootStorage.copyFile(oldPath,
"%s~%d" % (newPath, n))
576 """Return supported keys.
577 @return (iterable) List of keys usable in a dataset identifier"""
578 return iter(self.keyDict.keys())
581 """Return a dict of supported keys and their value types for a given dataset
582 type at a given level of the key hierarchy.
584 @param datasetType (str) dataset type or None for all dataset types
585 @param level (str) level or None for all levels or '' for the default level for the camera
586 @return (dict) dict keys are strings usable in a dataset identifier; values are their value types"""
592 if datasetType
is None:
593 keyDict = copy.copy(self.
keyDict)
596 if level
is not None and level
in self.
levels:
597 keyDict = copy.copy(keyDict)
598 for l
in self.
levels[level]:
613 """Return the name of the camera that this CameraMapper is for."""
615 className = className[className.find(
'.'):-1]
616 m = re.search(
r'(\w+)Mapper', className)
618 m = re.search(
r"class '[\w.]*?(\w+)'", className)
620 return name[:1].lower() + name[1:]
if name
else ''
624 """Return the name of the package containing this CameraMapper."""
625 if cls.packageName
is None:
626 raise ValueError(
'class variable packageName must not be None')
627 return cls.packageName
630 """Map a camera dataset."""
632 raise RuntimeError(
"No camera dataset available.")
634 return dafPersist.ButlerLocation(
635 pythonType=
"lsst.afw.cameraGeom.CameraConfig",
637 storageName=
"ConfigStorage",
645 """Return the (preloaded) camera object.
648 raise RuntimeError(
"No camera dataset available.")
652 """Map defects dataset.
654 @return a very minimal ButlerLocation containing just the locationList field
655 (just enough information that bypass_defects can use it).
658 if defectFitsPath
is None:
659 raise RuntimeError(
"No defects available for dataId=%s" % (dataId,))
661 return dafPersist.ButlerLocation(
None,
None,
None, defectFitsPath,
666 """Return a defect based on the butler location returned by map_defects
668 @param[in] butlerLocation: a ButlerLocation with locationList = path to defects FITS file
669 @param[in] dataId: the usual data ID; "ccd" must be set
671 Note: the name "bypass_XXX" means the butler makes no attempt to convert the ButlerLocation
672 into an object, which is what we want for now, since that conversion is a bit tricky.
675 defectsFitsPath = butlerLocation.locationList[0]
676 with pyfits.open(defectsFitsPath)
as hduList:
677 for hdu
in hduList[1:]:
678 if hdu.header[
"name"] != detectorName:
682 for data
in hdu.data:
683 bbox = afwGeom.Box2I(
684 afwGeom.Point2I(int(data[
'x0']), int(data[
'y0'])),
685 afwGeom.Extent2I(int(data[
'width']), int(data[
'height'])),
687 defectList.append(afwImage.DefectBase(bbox))
690 raise RuntimeError(
"No defects for ccd %s in %s" % (detectorName, defectsFitsPath))
693 return dafPersist.ButlerLocation(
694 pythonType=
"lsst.obs.base.ExposureIdInfo",
696 storageName=
"Internal",
697 locationList=
"ignored",
704 """Hook to retrieve an lsst.obs.base.ExposureIdInfo for an exposure"""
705 expId = self.bypass_ccdExposureId(datasetType, pythonType, location, dataId)
706 expBits = self.bypass_ccdExposureId_bits(datasetType, pythonType, location, dataId)
707 return ExposureIdInfo(expId=expId, expBits=expBits)
710 """Disable standardization for bfKernel
712 bfKernel is a calibration product that is numpy array,
713 unlike other calibration products that are all images;
714 all calibration images are sent through _standardizeExposure
715 due to CalibrationMapping, but we don't want that to happen to bfKernel
720 """Standardize a raw dataset by converting it to an Exposure instead of an Image"""
722 trimmed=
False, setVisitInfo=
True)
725 """Map a sky policy."""
726 return dafPersist.ButlerLocation(
"lsst.pex.policy.Policy",
"Policy",
727 "Internal",
None,
None, self,
731 """Standardize a sky policy by returning the one we use."""
740 def _getCcdKeyVal(self, dataId):
741 """Return CCD key and value used to look a defect in the defect registry
743 The default implementation simply returns ("ccd", full detector name)
747 def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True,
749 """Set up a registry (usually SQLite3), trying a number of possible
757 Description of registry (for log messages)
761 Policy that contains the registry name, used if path is None.
763 Key in policy for registry path.
764 storage : Storage subclass
765 Repository Storage to look in.
766 searchParents : bool, optional
767 True if the search for a registry should follow any Butler v1
769 posixIfNoSql : bool, optional
770 If an sqlite registry is not found, will create a posix registry if
775 lsst.daf.persistence.Registry
778 if path
is None and policyKey
in policy:
779 path = dafPersist.LogicalLocation(policy[policyKey]).locString()
780 if os.path.isabs(path):
781 raise RuntimeError(
"Policy should not indicate an absolute path for registry.")
782 if not storage.exists(path):
783 newPath = storage.instanceSearch(path)
785 newPath = newPath[0]
if newPath
is not None and len(newPath)
else None
787 self.log.warn(
"Unable to locate registry at policy path (also looked in root): %s",
791 self.log.warn(
"Unable to locate registry at policy path: %s", path)
799 if path
and (path.startswith(root)):
800 path = path[len(root +
'/'):]
801 except AttributeError:
807 def search(filename, description):
808 """Search for file in storage
813 Filename to search for
815 Description of file, for error message.
819 path : `str` or `None`
820 Path to file, or None
822 result = storage.instanceSearch(filename)
825 self.log.debug(
"Unable to locate %s: %s", description, filename)
830 path = search(
"%s.pgsql" % name,
"%s in root" % description)
832 path = search(
"%s.sqlite3" % name,
"%s in root" % description)
834 path = search(os.path.join(
".",
"%s.sqlite3" % name),
"%s in current dir" % description)
837 if not storage.exists(path):
838 newPath = storage.instanceSearch(path)
839 newPath = newPath[0]
if newPath
is not None and len(newPath)
else None
840 if newPath
is not None:
842 localFileObj = storage.getLocalFile(path)
843 self.log.info(
"Loading %s registry from %s", description, localFileObj.name)
844 registry = dafPersist.Registry.create(localFileObj.name)
846 elif not registry
and posixIfNoSql:
848 self.log.info(
"Loading Posix %s registry from %s", description, storage.root)
849 registry = dafPersist.PosixRegistry(storage.root)
855 def _transformId(self, dataId):
856 """Generate a standard ID dict from a camera-specific ID dict.
858 Canonical keys include:
859 - amp: amplifier name
860 - ccd: CCD name (in LSST this is a combination of raft and sensor)
861 The default implementation returns a copy of its input.
863 @param dataId[in] (dict) Dataset identifier; this must not be modified
864 @return (dict) Transformed dataset identifier"""
868 def _mapActualToPath(self, template, actualId):
869 """Convert a template path to an actual path, using the actual data
870 identifier. This implementation is usually sufficient but can be
871 overridden by the subclass.
872 @param template (string) Template path
873 @param actualId (dict) Dataset identifier
874 @return (string) Pathname"""
878 return template % transformedId
879 except Exception
as e:
880 raise RuntimeError(
"Failed to format %r with data %r: %s" % (template, transformedId, e))
884 """Convert a CCD name to a form useful as a filename
886 The default implementation converts spaces to underscores.
888 return ccdName.replace(
" ",
"_")
890 def _extractDetectorName(self, dataId):
891 """Extract the detector (CCD) name from the dataset identifier.
893 The name in question is the detector name used by lsst.afw.cameraGeom.
895 @param dataId (dict) Dataset identifier
896 @return (string) Detector name
898 raise NotImplementedError(
"No _extractDetectorName() function specified")
900 def _extractAmpId(self, dataId):
901 """Extract the amplifier identifer from a dataset identifier.
903 @warning this is deprecated; DO NOT USE IT
905 amplifier identifier has two parts: the detector name for the CCD
906 containing the amplifier and index of the amplifier in the detector.
907 @param dataId (dict) Dataset identifer
908 @return (tuple) Amplifier identifier"""
911 return (trDataId[
"ccd"], int(trDataId[
'amp']))
913 def _setAmpDetector(self, item, dataId, trimmed=True):
914 """Set the detector object in an Exposure for an amplifier.
915 Defects are also added to the Exposure based on the detector object.
916 @param[in,out] item (lsst.afw.image.Exposure)
917 @param dataId (dict) Dataset identifier
918 @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
922 def _setCcdDetector(self, item, dataId, trimmed=True):
923 """Set the detector object in an Exposure for a CCD.
924 @param[in,out] item (lsst.afw.image.Exposure)
925 @param dataId (dict) Dataset identifier
926 @param trimmed (bool) Should detector be marked as trimmed? (ignored)"""
928 if item.getDetector()
is not None:
932 detector = self.
camera[detectorName]
933 item.setDetector(detector)
935 def _setFilter(self, mapping, item, dataId):
936 """Set the filter object in an Exposure. If the Exposure had a FILTER
937 keyword, this was already processed during load. But if it didn't,
938 use the filter from the registry.
939 @param mapping (lsst.obs.base.Mapping)
940 @param[in,out] item (lsst.afw.image.Exposure)
941 @param dataId (dict) Dataset identifier"""
943 if not (isinstance(item, afwImage.ExposureU)
or isinstance(item, afwImage.ExposureI)
or
944 isinstance(item, afwImage.ExposureF)
or isinstance(item, afwImage.ExposureD)):
947 if item.getFilter().getId() != afwImage.Filter.UNKNOWN:
950 actualId = mapping.need([
'filter'], dataId)
951 filterName = actualId[
'filter']
953 filterName = self.
filters[filterName]
954 item.setFilter(afwImage.Filter(filterName))
957 def _standardizeExposure(self, mapping, item, dataId, filter=True,
958 trimmed=
True, setVisitInfo=
True):
959 """Default standardization function for images.
961 This sets the Detector from the camera geometry
962 and optionally set the Fiter. In both cases this saves
963 having to persist some data in each exposure (or image).
965 @param mapping (lsst.obs.base.Mapping)
966 @param[in,out] item image-like object; any of lsst.afw.image.Exposure,
967 lsst.afw.image.DecoratedImage, lsst.afw.image.Image
968 or lsst.afw.image.MaskedImage
969 @param dataId (dict) Dataset identifier
970 @param filter (bool) Set filter? Ignored if item is already an exposure
971 @param trimmed (bool) Should detector be marked as trimmed?
972 @param setVisitInfo (bool) Should Exposure have its VisitInfo filled out from the metadata?
973 @return (lsst.afw.image.Exposure) the standardized Exposure"""
976 except Exception
as e:
977 self.log.error(
"Could not turn item=%r into an exposure: %s" % (repr(item), e))
980 if mapping.level.lower() ==
"amp":
982 elif mapping.level.lower() ==
"ccd":
990 def _defectLookup(self, dataId):
991 """Find the defects for a given CCD.
992 @param dataId (dict) Dataset identifier
993 @return (string) path to the defects file or None if not available"""
997 raise RuntimeError(
"No registry for defect lookup")
1001 dataIdForLookup = {
'visit': dataId[
'visit']}
1003 rows = self.registry.lookup((
'taiObs'), (
'raw_visit'), dataIdForLookup)
1006 assert len(rows) == 1
1010 rows = self.defectRegistry.executeQuery((
"path",), (
"defect",),
1012 (
"DATETIME(?)",
"DATETIME(validStart)",
"DATETIME(validEnd)"),
1014 if not rows
or len(rows) == 0:
1017 return os.path.join(self.
defectPath, rows[0][0])
1019 raise RuntimeError(
"Querying for defects (%s, %s) returns %d files: %s" %
1020 (ccdVal, taiObs, len(rows),
", ".join([_[0]
for _
in rows])))
1022 def _makeCamera(self, policy, repositoryDir):
1023 """Make a camera (instance of lsst.afw.cameraGeom.Camera) describing the camera geometry
1025 Also set self.cameraDataLocation, if relevant (else it can be left None).
1027 This implementation assumes that policy contains an entry "camera" that points to the
1028 subdirectory in this package of camera data; specifically, that subdirectory must contain:
1029 - a file named `camera.py` that contains persisted camera config
1030 - ampInfo table FITS files, as required by lsst.afw.cameraGeom.makeCameraFromPath
1032 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility))
1033 Policy with per-camera defaults already merged
1034 @param repositoryDir (string) Policy repository for the subclassing
1035 module (obtained with getRepositoryPath() on the
1036 per-camera default dictionary)
1038 if isinstance(policy, pexPolicy.Policy):
1039 policy = dafPersist.Policy(pexPolicy=policy)
1040 if 'camera' not in policy:
1041 raise RuntimeError(
"Cannot find 'camera' in policy; cannot construct a camera")
1042 cameraDataSubdir = policy[
'camera']
1044 os.path.join(repositoryDir, cameraDataSubdir,
"camera.py"))
1045 cameraConfig = afwCameraGeom.CameraConfig()
1048 return afwCameraGeom.makeCameraFromPath(
1049 cameraConfig=cameraConfig,
1050 ampInfoPath=ampInfoPath,
1056 """Get the registry used by this mapper.
1061 The registry used by this mapper for this mapper's repository.
1067 """Generate an Exposure from an image-like object
1069 If the image is a DecoratedImage then also set its WCS and metadata
1070 (Image and MaskedImage are missing the necessary metadata
1071 and Exposure already has those set)
1073 @param[in] image Image-like object (lsst.afw.image.DecoratedImage, Image, MaskedImage or Exposure)
1074 @return (lsst.afw.image.Exposure) Exposure containing input image
1077 if isinstance(image, afwImage.MaskedImage):
1078 exposure = afwImage.makeExposure(image)
1079 elif isinstance(image, afwImage.DecoratedImage):
1080 exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image.getImage()))
1081 metadata = image.getMetadata()
1083 wcs = afwImage.makeWcs(metadata,
True)
1084 exposure.setWcs(wcs)
1085 except pexExcept.InvalidParameterError
as e:
1088 logger = lsstLog.Log.getLogger(
"CameraMapper")
1089 logger.warn(
"wcs set to None; insufficient information found in metadata to create a valid wcs: "
1092 exposure.setMetadata(metadata)
1093 elif isinstance(image, afwImage.Exposure):
1096 metadata = exposure.getMetadata()
1099 exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image))
1103 if setVisitInfo
and exposure.getInfo().getVisitInfo()
is None:
1104 if metadata
is not None:
1107 logger = lsstLog.Log.getLogger(
"CameraMapper")
1108 logger.warn(
"I can only set the VisitInfo if you provide a mapper")
1110 exposureId = mapper._computeCcdExposureId(dataId)
1111 visitInfo = mapper.makeRawVisitInfo(md=metadata, exposureId=exposureId)
1113 exposure.getInfo().setVisitInfo(visitInfo)
def _getCcdKeyVal
Utility functions.