23 from builtins
import str
29 import lsst.daf.persistence
as dafPersist
30 from .
import ImageMapping, ExposureMapping, CalibrationMapping, DatasetMapping
31 import lsst.afw.geom
as afwGeom
32 import lsst.afw.image
as afwImage
33 import lsst.afw.table
as afwTable
34 import lsst.afw.cameraGeom
as afwCameraGeom
35 import lsst.log
as lsstLog
36 import lsst.pex.policy
as pexPolicy
37 import lsst.pex.exceptions
as pexExcept
38 from .exposureIdInfo
import ExposureIdInfo
39 from .makeRawVisitInfo
import MakeRawVisitInfo
40 from lsst.utils
import getPackageDir
42 """This module defines the CameraMapper base class.""" 47 """CameraMapper is a base class for mappers that handle images from a 48 camera and products derived from them. This provides an abstraction layer 49 between the data on disk and the code. 51 Public methods: keys, queryMetadata, getDatasetTypes, map, 52 canStandardize, standardize 54 Mappers for specific data sources (e.g., CFHT Megacam, LSST 55 simulations, etc.) should inherit this class. 57 The CameraMapper manages datasets within a "root" directory. Note that 58 writing to a dataset present in the input root will hide the existing 59 dataset but not overwrite it. See #2160 for design discussion. 61 A camera is assumed to consist of one or more rafts, each composed of 62 multiple CCDs. Each CCD is in turn composed of one or more amplifiers 63 (amps). A camera is also assumed to have a camera geometry description 64 (CameraGeom object) as a policy file, a filter description (Filter class 65 static configuration) as another policy file, and an optional defects 66 description directory. 68 Information from the camera geometry and defects are inserted into all 69 Exposure objects returned. 71 The mapper uses one or two registries to retrieve metadata about the 72 images. The first is a registry of all raw exposures. This must contain 73 the time of the observation. One or more tables (or the equivalent) 74 within the registry are used to look up data identifier components that 75 are not specified by the user (e.g. filter) and to return results for 76 metadata queries. The second is an optional registry of all calibration 77 data. This should contain validity start and end entries for each 78 calibration dataset in the same timescale as the observation time. 80 Subclasses will typically set MakeRawVisitInfoClass: 82 MakeRawVisitInfoClass: a class variable that points to a subclass of 83 MakeRawVisitInfo, a functor that creates an 84 lsst.afw.image.VisitInfo from the FITS metadata of a raw image. 86 Subclasses must provide the following methods: 88 _extractDetectorName(self, dataId): returns the detector name for a CCD 89 (e.g., "CFHT 21", "R:1,2 S:3,4") as used in the AFW CameraGeom class given 90 a dataset identifier referring to that CCD or a subcomponent of it. 92 _computeCcdExposureId(self, dataId): see below 94 _computeCoaddExposureId(self, dataId, singleFilter): see below 96 Subclasses may also need to override the following methods: 98 _transformId(self, dataId): transformation of a data identifier 99 from colloquial usage (e.g., "ccdname") to proper/actual usage (e.g., "ccd"), 100 including making suitable for path expansion (e.g. removing commas). 101 The default implementation does nothing. Note that this 102 method should not modify its input parameter. 104 getShortCcdName(self, ccdName): a static method that returns a shortened name 105 suitable for use as a filename. The default version converts spaces to underscores. 107 _getCcdKeyVal(self, dataId): return a CCD key and value 108 by which to look up defects in the defects registry. 109 The default value returns ("ccd", detector name) 111 _mapActualToPath(self, template, actualId): convert a template path to an 112 actual path, using the actual dataset identifier. 114 The mapper's behaviors are largely specified by the policy file. 115 See the MapperDictionary.paf for descriptions of the available items. 117 The 'exposures', 'calibrations', and 'datasets' subpolicies configure 118 mappings (see Mappings class). 120 Common default mappings for all subclasses can be specified in the 121 "policy/{images,exposures,calibrations,datasets}.yaml" files. This provides 122 a simple way to add a product to all camera mappers. 124 Functions to map (provide a path to the data given a dataset 125 identifier dictionary) and standardize (convert data into some standard 126 format or type) may be provided in the subclass as "map_{dataset type}" 127 and "std_{dataset type}", respectively. 129 If non-Exposure datasets cannot be retrieved using standard 130 daf_persistence methods alone, a "bypass_{dataset type}" function may be 131 provided in the subclass to return the dataset instead of using the 132 "datasets" subpolicy. 134 Implementations of map_camera and bypass_camera that should typically be 135 sufficient are provided in this base class. 138 * Handle defects the same was as all other calibration products, using the calibration registry 139 * Instead of auto-loading the camera at construction time, load it from the calibration registry 140 * Rewrite defects as AFW tables so we don't need pyfits to unpersist them; then remove all mention 141 of pyfits from this package. 147 MakeRawVisitInfoClass = MakeRawVisitInfo
150 PupilFactoryClass = afwCameraGeom.PupilFactory
152 def __init__(self, policy, repositoryDir,
153 root=None, registry=None, calibRoot=None, calibRegistry=None,
154 provided=None, parentRegistry=None, repositoryCfg=None):
155 """Initialize the CameraMapper. 159 policy : daf_persistence.Policy, 160 Can also be pexPolicy.Policy, only for backward compatibility. 161 Policy with per-camera defaults already merged. 162 repositoryDir : string 163 Policy repository for the subclassing module (obtained with 164 getRepositoryPath() on the per-camera default dictionary). 165 root : string, optional 166 Path to the root directory for data. 167 registry : string, optional 168 Path to registry with data's metadata. 169 calibRoot : string, optional 170 Root directory for calibrations. 171 calibRegistry : string, optional 172 Path to registry with calibrations' metadata. 173 provided : list of string, optional 174 Keys provided by the mapper. 175 parentRegistry : Registry subclass, optional 176 Registry from a parent repository that may be used to look up 178 repositoryCfg : daf_persistence.RepositoryCfg or None, optional 179 The configuration information for the repository this mapper is 183 dafPersist.Mapper.__init__(self)
185 self.
log = lsstLog.Log.getLogger(
"CameraMapper")
190 self.
root = repositoryCfg.root
193 if isinstance(policy, pexPolicy.Policy):
194 policy = dafPersist.Policy(policy)
196 repoPolicy = repositoryCfg.policy
if repositoryCfg
else None 197 if repoPolicy
is not None:
198 policy.update(repoPolicy)
200 defaultPolicyFile = dafPersist.Policy.defaultPolicyFile(
"obs_base",
201 "MapperDictionary.paf",
203 dictPolicy = dafPersist.Policy(defaultPolicyFile)
204 policy.merge(dictPolicy)
208 if 'levels' in policy:
209 levelsPolicy = policy[
'levels']
210 for key
in levelsPolicy.names(
True):
211 self.
levels[key] = set(levelsPolicy.asArray(key))
214 if 'defaultSubLevels' in policy:
220 root = dafPersist.LogicalLocation(root).locString()
230 if calibRoot
is not None:
231 calibRoot = dafPersist.Storage.absolutePath(root, calibRoot)
232 calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
235 calibRoot = policy.get(
'calibRoot',
None)
237 calibStorage = dafPersist.Storage.makeFromURI(uri=calibRoot,
239 if calibStorage
is None:
247 posixIfNoSql=(
not parentRegistry))
250 needCalibRegistry = policy.get(
'needCalibRegistry',
None)
251 if needCalibRegistry:
254 "calibRegistryPath", calibStorage)
257 "'needCalibRegistry' is true in Policy, but was unable to locate a repo at " +
258 "calibRoot ivar:%s or policy['calibRoot']:%s" %
259 (calibRoot, policy.get(
'calibRoot',
None)))
275 if 'defects' in policy:
276 self.
defectPath = os.path.join(repositoryDir, policy[
'defects'])
277 defectRegistryLocation = os.path.join(self.
defectPath,
"defectRegistry.sqlite3")
278 self.
defectRegistry = dafPersist.Registry.create(defectRegistryLocation)
289 raise ValueError(
'class variable packageName must not be None')
293 def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None):
294 """Initialize mappings 296 For each of the dataset types that we want to be able to read, there are 297 methods that can be created to support them: 298 * map_<dataset> : determine the path for dataset 299 * std_<dataset> : standardize the retrieved dataset 300 * bypass_<dataset> : retrieve the dataset (bypassing the usual retrieval machinery) 301 * query_<dataset> : query the registry 303 Besides the dataset types explicitly listed in the policy, we create 304 additional, derived datasets for additional conveniences, e.g., reading 305 the header of an image, retrieving only the size of a catalog. 307 @param policy (Policy) Policy with per-camera defaults already merged 308 @param rootStorage (Storage subclass instance) Interface to persisted repository data 309 @param calibRoot (Storage subclass instance) Interface to persisted calib repository data 310 @param provided (list of strings) Keys provided by the mapper 313 imgMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
314 "obs_base",
"ImageMappingDictionary.paf",
"policy"))
315 expMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
316 "obs_base",
"ExposureMappingDictionary.paf",
"policy"))
317 calMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
318 "obs_base",
"CalibrationMappingDictionary.paf",
"policy"))
319 dsMappingPolicy = dafPersist.Policy(dafPersist.Policy.defaultPolicyFile(
320 "obs_base",
"DatasetMappingDictionary.paf",
"policy"))
324 (
"images", imgMappingPolicy, ImageMapping),
325 (
"exposures", expMappingPolicy, ExposureMapping),
326 (
"calibrations", calMappingPolicy, CalibrationMapping),
327 (
"datasets", dsMappingPolicy, DatasetMapping)
330 for name, defPolicy, cls
in mappingList:
332 datasets = policy[name]
335 defaultsPath = os.path.join(getPackageDir(
"obs_base"),
"policy", name +
".yaml")
336 if os.path.exists(defaultsPath):
337 datasets.merge(dafPersist.Policy(defaultsPath))
340 setattr(self, name, mappings)
341 for datasetType
in datasets.names(
True):
342 subPolicy = datasets[datasetType]
343 subPolicy.merge(defPolicy)
345 if not hasattr(self,
"map_" + datasetType)
and 'composite' in subPolicy:
346 def compositeClosure(dataId, write=False, mapper=None, mapping=None, subPolicy=subPolicy):
347 components = subPolicy.get(
'composite')
348 assembler = subPolicy[
'assembler']
if 'assembler' in subPolicy
else None 349 disassembler = subPolicy[
'disassembler']
if 'disassembler' in subPolicy
else None 350 python = subPolicy[
'python']
351 butlerComposite = dafPersist.ButlerComposite(assembler=assembler,
352 disassembler=disassembler,
356 for name, component
in components.items():
357 butlerComposite.add(id=name,
358 datasetType=component.get(
'datasetType'),
359 setter=component.get(
'setter',
None),
360 getter=component.get(
'getter',
None),
361 subset=component.get(
'subset',
False),
362 inputOnly=component.get(
'inputOnly',
False))
363 return butlerComposite
364 setattr(self,
"map_" + datasetType, compositeClosure)
368 if name ==
"calibrations":
370 provided=provided, dataRoot=rootStorage)
372 mapping = cls(datasetType, subPolicy, self.
registry, rootStorage, provided=provided)
373 self.
keyDict.update(mapping.keys())
374 mappings[datasetType] = mapping
375 self.
mappings[datasetType] = mapping
376 if not hasattr(self,
"map_" + datasetType):
377 def mapClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
378 return mapping.map(mapper, dataId, write)
379 setattr(self,
"map_" + datasetType, mapClosure)
380 if not hasattr(self,
"query_" + datasetType):
381 def queryClosure(format, dataId, mapping=mapping):
382 return mapping.lookup(format, dataId)
383 setattr(self,
"query_" + datasetType, queryClosure)
384 if hasattr(mapping,
"standardize")
and not hasattr(self,
"std_" + datasetType):
385 def stdClosure(item, dataId, mapper=weakref.proxy(self), mapping=mapping):
386 return mapping.standardize(mapper, item, dataId)
387 setattr(self,
"std_" + datasetType, stdClosure)
389 def setMethods(suffix, mapImpl=None, bypassImpl=None, queryImpl=None):
390 """Set convenience methods on CameraMapper""" 391 mapName =
"map_" + datasetType +
"_" + suffix
392 bypassName =
"bypass_" + datasetType +
"_" + suffix
393 queryName =
"query_" + datasetType +
"_" + suffix
394 if not hasattr(self, mapName):
395 setattr(self, mapName, mapImpl
or getattr(self,
"map_" + datasetType))
396 if not hasattr(self, bypassName):
397 if bypassImpl
is None and hasattr(self,
"bypass_" + datasetType):
398 bypassImpl = getattr(self,
"bypass_" + datasetType)
399 if bypassImpl
is not None:
400 setattr(self, bypassName, bypassImpl)
401 if not hasattr(self, queryName):
402 setattr(self, queryName, queryImpl
or getattr(self,
"query_" + datasetType))
405 setMethods(
"filename", bypassImpl=
lambda datasetType, pythonType, location, dataId:
406 [os.path.join(location.getStorage().root, p)
for p
in location.getLocations()])
408 if subPolicy[
"storage"] ==
"FitsStorage":
409 setMethods(
"md", bypassImpl=
lambda datasetType, pythonType, location, dataId:
410 afwImage.readMetadata(location.getLocationsWithRoot()[0]))
411 if name ==
"exposures":
412 setMethods(
"wcs", bypassImpl=
lambda datasetType, pythonType, location, dataId:
414 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
415 setMethods(
"calib", bypassImpl=
lambda datasetType, pythonType, location, dataId:
417 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
418 setMethods(
"visitInfo",
419 bypassImpl=
lambda datasetType, pythonType, location, dataId:
421 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
423 bypassImpl=
lambda datasetType, pythonType, location, dataId:
425 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
426 setMethods(
"detector",
427 mapImpl=
lambda dataId, write=
False:
428 dafPersist.ButlerLocation(
429 pythonType=
"lsst.afw.cameraGeom.CameraConfig",
431 storageName=
"Internal",
432 locationList=
"ignored",
437 bypassImpl=
lambda datasetType, pythonType, location, dataId:
440 setMethods(
"bbox", bypassImpl=
lambda dsType, pyType, location, dataId:
441 afwImage.bboxFromMetadata(
442 afwImage.readMetadata(location.getLocationsWithRoot()[0], hdu=1)))
444 elif name ==
"images":
445 setMethods(
"bbox", bypassImpl=
lambda dsType, pyType, location, dataId:
446 afwImage.bboxFromMetadata(
447 afwImage.readMetadata(location.getLocationsWithRoot()[0])))
449 if subPolicy[
"storage"] ==
"FitsCatalogStorage":
450 setMethods(
"md", bypassImpl=
lambda datasetType, pythonType, location, dataId:
451 afwImage.readMetadata(os.path.join(location.getStorage().root,
452 location.getLocations()[0]), hdu=1))
455 if subPolicy[
"storage"] ==
"FitsStorage":
456 def mapSubClosure(dataId, write=False, mapper=weakref.proxy(self), mapping=mapping):
457 subId = dataId.copy()
459 loc = mapping.map(mapper, subId, write)
460 bbox = dataId[
'bbox']
461 llcX = bbox.getMinX()
462 llcY = bbox.getMinY()
463 width = bbox.getWidth()
464 height = bbox.getHeight()
465 loc.additionalData.set(
'llcX', llcX)
466 loc.additionalData.set(
'llcY', llcY)
467 loc.additionalData.set(
'width', width)
468 loc.additionalData.set(
'height', height)
469 if 'imageOrigin' in dataId:
470 loc.additionalData.set(
'imageOrigin',
471 dataId[
'imageOrigin'])
473 def querySubClosure(key, format, dataId, mapping=mapping):
474 subId = dataId.copy()
476 return mapping.lookup(format, subId)
477 setMethods(
"sub", mapImpl=mapSubClosure, queryImpl=querySubClosure)
479 if subPolicy[
"storage"] ==
"FitsCatalogStorage":
481 setMethods(
"len", bypassImpl=
lambda datasetType, pythonType, location, dataId:
482 afwImage.readMetadata(os.path.join(location.getStorage().root,
483 location.getLocations()[0]),
484 hdu=1).get(
"NAXIS2"))
487 if not datasetType.endswith(
"_schema")
and datasetType +
"_schema" not in datasets:
488 setMethods(
"schema", bypassImpl=
lambda datasetType, pythonType, location, dataId:
489 afwTable.Schema.readFits(os.path.join(location.getStorage().root,
490 location.getLocations()[0])))
492 def _computeCcdExposureId(self, dataId):
493 """Compute the 64-bit (long) identifier for a CCD exposure. 495 Subclasses must override 497 @param dataId (dict) Data identifier with visit, ccd 499 raise NotImplementedError()
501 def _computeCoaddExposureId(self, dataId, singleFilter):
502 """Compute the 64-bit (long) identifier for a coadd. 504 Subclasses must override 506 @param dataId (dict) Data identifier with tract and patch. 507 @param singleFilter (bool) True means the desired ID is for a single- 508 filter coadd, in which case dataId 511 raise NotImplementedError()
513 def _search(self, path):
514 """Search for path in the associated repository's storage. 519 Path that describes an object in the repository associated with 521 Path may contain an HDU indicator, e.g. 'foo.fits[1]'. The 522 indicator will be stripped when searching and so will match 523 filenames without the HDU indicator, e.g. 'foo.fits'. The path 524 returned WILL contain the indicator though, e.g. ['foo.fits[1]']. 529 The path for this object in the repository. Will return None if the 530 object can't be found. If the input argument path contained an HDU 531 indicator, the returned path will also contain the HDU indicator. 536 """Rename any existing object with the given type and dataId. 538 The CameraMapper implementation saves objects in a sequence of e.g.: 542 All of the backups will be placed in the output repo, however, and will 543 not be removed if they are found elsewhere in the _parent chain. This 544 means that the same file will be stored twice if the previous version was 545 found in an input repo. 554 def firstElement(list):
555 """Get the first element in the list, or None if that can't be done. 557 return list[0]
if list
is not None and len(list)
else None 560 newLocation = self.map(datasetType, dataId, write=
True)
561 newPath = newLocation.getLocations()[0]
562 path = dafPersist.PosixStorage.search(self.
root, newPath, searchParents=
True)
563 path = firstElement(path)
565 while path
is not None:
567 oldPaths.append((n, path))
568 path = dafPersist.PosixStorage.search(self.
root,
"%s~%d" % (newPath, n), searchParents=
True)
569 path = firstElement(path)
570 for n, oldPath
in reversed(oldPaths):
571 self.
rootStorage.copyFile(oldPath,
"%s~%d" % (newPath, n))
574 """Return supported keys. 575 @return (iterable) List of keys usable in a dataset identifier""" 579 """Return a dict of supported keys and their value types for a given dataset 580 type at a given level of the key hierarchy. 582 @param datasetType (str) dataset type or None for all dataset types 583 @param level (str) level or None for all levels or '' for the default level for the camera 584 @return (dict) dict keys are strings usable in a dataset identifier; values are their value types""" 590 if datasetType
is None:
591 keyDict = copy.copy(self.
keyDict)
594 if level
is not None and level
in self.
levels:
595 keyDict = copy.copy(keyDict)
596 for l
in self.
levels[level]:
611 """Return the name of the camera that this CameraMapper is for.""" 613 className = className[className.find(
'.'):-1]
614 m = re.search(
r'(\w+)Mapper', className)
616 m = re.search(
r"class '[\w.]*?(\w+)'", className)
618 return name[:1].lower() + name[1:]
if name
else '' 622 """Return the name of the package containing this CameraMapper.""" 624 raise ValueError(
'class variable packageName must not be None')
628 """Map a camera dataset.""" 630 raise RuntimeError(
"No camera dataset available.")
632 return dafPersist.ButlerLocation(
633 pythonType=
"lsst.afw.cameraGeom.CameraConfig",
635 storageName=
"ConfigStorage",
643 """Return the (preloaded) camera object. 646 raise RuntimeError(
"No camera dataset available.")
650 """Map defects dataset. 652 @return a very minimal ButlerLocation containing just the locationList field 653 (just enough information that bypass_defects can use it). 656 if defectFitsPath
is None:
657 raise RuntimeError(
"No defects available for dataId=%s" % (dataId,))
659 return dafPersist.ButlerLocation(
None,
None,
None, defectFitsPath,
664 """Return a defect based on the butler location returned by map_defects 666 @param[in] butlerLocation: a ButlerLocation with locationList = path to defects FITS file 667 @param[in] dataId: the usual data ID; "ccd" must be set 669 Note: the name "bypass_XXX" means the butler makes no attempt to convert the ButlerLocation 670 into an object, which is what we want for now, since that conversion is a bit tricky. 673 defectsFitsPath = butlerLocation.locationList[0]
674 with pyfits.open(defectsFitsPath)
as hduList:
675 for hdu
in hduList[1:]:
676 if hdu.header[
"name"] != detectorName:
680 for data
in hdu.data:
681 bbox = afwGeom.Box2I(
682 afwGeom.Point2I(int(data[
'x0']), int(data[
'y0'])),
683 afwGeom.Extent2I(int(data[
'width']), int(data[
'height'])),
685 defectList.append(afwImage.DefectBase(bbox))
688 raise RuntimeError(
"No defects for ccd %s in %s" % (detectorName, defectsFitsPath))
691 return dafPersist.ButlerLocation(
692 pythonType=
"lsst.obs.base.ExposureIdInfo",
694 storageName=
"Internal",
695 locationList=
"ignored",
702 """Hook to retrieve an lsst.obs.base.ExposureIdInfo for an exposure""" 703 expId = self.bypass_ccdExposureId(datasetType, pythonType, location, dataId)
704 expBits = self.bypass_ccdExposureId_bits(datasetType, pythonType, location, dataId)
708 """Disable standardization for bfKernel 710 bfKernel is a calibration product that is numpy array, 711 unlike other calibration products that are all images; 712 all calibration images are sent through _standardizeExposure 713 due to CalibrationMapping, but we don't want that to happen to bfKernel 718 """Standardize a raw dataset by converting it to an Exposure instead of an Image""" 720 trimmed=
False, setVisitInfo=
True)
723 """Map a sky policy.""" 724 return dafPersist.ButlerLocation(
"lsst.pex.policy.Policy",
"Policy",
725 "Internal",
None,
None, self,
729 """Standardize a sky policy by returning the one we use.""" 738 def _getCcdKeyVal(self, dataId):
739 """Return CCD key and value used to look a defect in the defect registry 741 The default implementation simply returns ("ccd", full detector name) 745 def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True,
747 """Set up a registry (usually SQLite3), trying a number of possible 755 Description of registry (for log messages) 759 Policy that contains the registry name, used if path is None. 761 Key in policy for registry path. 762 storage : Storage subclass 763 Repository Storage to look in. 764 searchParents : bool, optional 765 True if the search for a registry should follow any Butler v1 767 posixIfNoSql : bool, optional 768 If an sqlite registry is not found, will create a posix registry if 773 lsst.daf.persistence.Registry 776 if path
is None and policyKey
in policy:
777 path = dafPersist.LogicalLocation(policy[policyKey]).locString()
778 if os.path.isabs(path):
779 raise RuntimeError(
"Policy should not indicate an absolute path for registry.")
780 if not storage.exists(path):
781 newPath = storage.instanceSearch(path)
783 newPath = newPath[0]
if newPath
is not None and len(newPath)
else None 785 self.
log.warn(
"Unable to locate registry at policy path (also looked in root): %s",
789 self.
log.warn(
"Unable to locate registry at policy path: %s", path)
797 if path
and (path.startswith(root)):
798 path = path[len(root +
'/'):]
799 except AttributeError:
805 def search(filename, description):
806 """Search for file in storage 811 Filename to search for 813 Description of file, for error message. 817 path : `str` or `None` 818 Path to file, or None 820 result = storage.instanceSearch(filename)
823 self.
log.debug(
"Unable to locate %s: %s", description, filename)
828 path = search(
"%s.pgsql" % name,
"%s in root" % description)
830 path = search(
"%s.sqlite3" % name,
"%s in root" % description)
832 path = search(os.path.join(
".",
"%s.sqlite3" % name),
"%s in current dir" % description)
835 if not storage.exists(path):
836 newPath = storage.instanceSearch(path)
837 newPath = newPath[0]
if newPath
is not None and len(newPath)
else None 838 if newPath
is not None:
840 localFileObj = storage.getLocalFile(path)
841 self.
log.info(
"Loading %s registry from %s", description, localFileObj.name)
842 registry = dafPersist.Registry.create(localFileObj.name)
843 elif not registry
and posixIfNoSql:
845 self.
log.info(
"Loading Posix %s registry from %s", description, storage.root)
846 registry = dafPersist.PosixRegistry(storage.root)
852 def _transformId(self, dataId):
853 """Generate a standard ID dict from a camera-specific ID dict. 855 Canonical keys include: 856 - amp: amplifier name 857 - ccd: CCD name (in LSST this is a combination of raft and sensor) 858 The default implementation returns a copy of its input. 860 @param dataId[in] (dict) Dataset identifier; this must not be modified 861 @return (dict) Transformed dataset identifier""" 865 def _mapActualToPath(self, template, actualId):
866 """Convert a template path to an actual path, using the actual data 867 identifier. This implementation is usually sufficient but can be 868 overridden by the subclass. 869 @param template (string) Template path 870 @param actualId (dict) Dataset identifier 871 @return (string) Pathname""" 875 return template % transformedId
876 except Exception
as e:
877 raise RuntimeError(
"Failed to format %r with data %r: %s" % (template, transformedId, e))
881 """Convert a CCD name to a form useful as a filename 883 The default implementation converts spaces to underscores. 885 return ccdName.replace(
" ",
"_")
887 def _extractDetectorName(self, dataId):
888 """Extract the detector (CCD) name from the dataset identifier. 890 The name in question is the detector name used by lsst.afw.cameraGeom. 892 @param dataId (dict) Dataset identifier 893 @return (string) Detector name 895 raise NotImplementedError(
"No _extractDetectorName() function specified")
897 def _extractAmpId(self, dataId):
898 """Extract the amplifier identifer from a dataset identifier. 900 @warning this is deprecated; DO NOT USE IT 902 amplifier identifier has two parts: the detector name for the CCD 903 containing the amplifier and index of the amplifier in the detector. 904 @param dataId (dict) Dataset identifer 905 @return (tuple) Amplifier identifier""" 908 return (trDataId[
"ccd"], int(trDataId[
'amp']))
910 def _setAmpDetector(self, item, dataId, trimmed=True):
911 """Set the detector object in an Exposure for an amplifier. 912 Defects are also added to the Exposure based on the detector object. 913 @param[in,out] item (lsst.afw.image.Exposure) 914 @param dataId (dict) Dataset identifier 915 @param trimmed (bool) Should detector be marked as trimmed? (ignored)""" 919 def _setCcdDetector(self, item, dataId, trimmed=True):
920 """Set the detector object in an Exposure for a CCD. 921 @param[in,out] item (lsst.afw.image.Exposure) 922 @param dataId (dict) Dataset identifier 923 @param trimmed (bool) Should detector be marked as trimmed? (ignored)""" 925 if item.getDetector()
is not None:
929 detector = self.
camera[detectorName]
930 item.setDetector(detector)
932 def _setFilter(self, mapping, item, dataId):
933 """Set the filter object in an Exposure. If the Exposure had a FILTER 934 keyword, this was already processed during load. But if it didn't, 935 use the filter from the registry. 936 @param mapping (lsst.obs.base.Mapping) 937 @param[in,out] item (lsst.afw.image.Exposure) 938 @param dataId (dict) Dataset identifier""" 940 if not (isinstance(item, afwImage.ExposureU)
or isinstance(item, afwImage.ExposureI)
or 941 isinstance(item, afwImage.ExposureF)
or isinstance(item, afwImage.ExposureD)):
944 if item.getFilter().getId() != afwImage.Filter.UNKNOWN:
947 actualId = mapping.need([
'filter'], dataId)
948 filterName = actualId[
'filter']
950 filterName = self.
filters[filterName]
951 item.setFilter(afwImage.Filter(filterName))
954 def _standardizeExposure(self, mapping, item, dataId, filter=True,
955 trimmed=True, setVisitInfo=True):
956 """Default standardization function for images. 958 This sets the Detector from the camera geometry 959 and optionally set the Fiter. In both cases this saves 960 having to persist some data in each exposure (or image). 962 @param mapping (lsst.obs.base.Mapping) 963 @param[in,out] item image-like object; any of lsst.afw.image.Exposure, 964 lsst.afw.image.DecoratedImage, lsst.afw.image.Image 965 or lsst.afw.image.MaskedImage 966 @param dataId (dict) Dataset identifier 967 @param filter (bool) Set filter? Ignored if item is already an exposure 968 @param trimmed (bool) Should detector be marked as trimmed? 969 @param setVisitInfo (bool) Should Exposure have its VisitInfo filled out from the metadata? 970 @return (lsst.afw.image.Exposure) the standardized Exposure""" 973 except Exception
as e:
974 self.
log.error(
"Could not turn item=%r into an exposure: %s" % (repr(item), e))
977 if mapping.level.lower() ==
"amp":
979 elif mapping.level.lower() ==
"ccd":
987 def _defectLookup(self, dataId):
988 """Find the defects for a given CCD. 989 @param dataId (dict) Dataset identifier 990 @return (string) path to the defects file or None if not available""" 994 raise RuntimeError(
"No registry for defect lookup")
998 dataIdForLookup = {
'visit': dataId[
'visit']}
1000 rows = self.
registry.lookup((
'taiObs'), (
'raw_visit'), dataIdForLookup)
1003 assert len(rows) == 1
1009 (
"DATETIME(?)",
"DATETIME(validStart)",
"DATETIME(validEnd)"),
1011 if not rows
or len(rows) == 0:
1014 return os.path.join(self.
defectPath, rows[0][0])
1016 raise RuntimeError(
"Querying for defects (%s, %s) returns %d files: %s" %
1017 (ccdVal, taiObs, len(rows),
", ".join([_[0]
for _
in rows])))
1019 def _makeCamera(self, policy, repositoryDir):
1020 """Make a camera (instance of lsst.afw.cameraGeom.Camera) describing the camera geometry 1022 Also set self.cameraDataLocation, if relevant (else it can be left None). 1024 This implementation assumes that policy contains an entry "camera" that points to the 1025 subdirectory in this package of camera data; specifically, that subdirectory must contain: 1026 - a file named `camera.py` that contains persisted camera config 1027 - ampInfo table FITS files, as required by lsst.afw.cameraGeom.makeCameraFromPath 1029 @param policy (daf_persistence.Policy, or pexPolicy.Policy (only for backward compatibility)) 1030 Policy with per-camera defaults already merged 1031 @param repositoryDir (string) Policy repository for the subclassing 1032 module (obtained with getRepositoryPath() on the 1033 per-camera default dictionary) 1035 if isinstance(policy, pexPolicy.Policy):
1036 policy = dafPersist.Policy(pexPolicy=policy)
1037 if 'camera' not in policy:
1038 raise RuntimeError(
"Cannot find 'camera' in policy; cannot construct a camera")
1039 cameraDataSubdir = policy[
'camera']
1041 os.path.join(repositoryDir, cameraDataSubdir,
"camera.py"))
1042 cameraConfig = afwCameraGeom.CameraConfig()
1045 return afwCameraGeom.makeCameraFromPath(
1046 cameraConfig=cameraConfig,
1047 ampInfoPath=ampInfoPath,
1053 """Get the registry used by this mapper. 1058 The registry used by this mapper for this mapper's repository. 1063 """Generate an Exposure from an image-like object 1065 If the image is a DecoratedImage then also set its WCS and metadata 1066 (Image and MaskedImage are missing the necessary metadata 1067 and Exposure already has those set) 1069 @param[in] image Image-like object (lsst.afw.image.DecoratedImage, Image, MaskedImage or Exposure) 1070 @return (lsst.afw.image.Exposure) Exposure containing input image 1073 if isinstance(image, afwImage.MaskedImage):
1074 exposure = afwImage.makeExposure(image)
1075 elif isinstance(image, afwImage.DecoratedImage):
1076 exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image.getImage()))
1077 metadata = image.getMetadata()
1079 wcs = afwImage.makeWcs(metadata,
True)
1080 exposure.setWcs(wcs)
1081 except pexExcept.InvalidParameterError
as e:
1084 logger = lsstLog.Log.getLogger(
"CameraMapper")
1085 logger.warn(
"wcs set to None; insufficient information found in metadata to create a valid wcs: " 1088 exposure.setMetadata(metadata)
1089 elif isinstance(image, afwImage.Exposure):
1092 metadata = exposure.getMetadata()
1095 exposure = afwImage.makeExposure(afwImage.makeMaskedImage(image))
1099 if setVisitInfo
and exposure.getInfo().getVisitInfo()
is None:
1100 if metadata
is not None:
1103 logger = lsstLog.Log.getLogger(
"CameraMapper")
1104 logger.warn(
"I can only set the VisitInfo if you provide a mapper")
1106 exposureId = mapper._computeCcdExposureId(dataId)
1107 visitInfo = mapper.makeRawVisitInfo(md=metadata, exposureId=exposureId)
1109 exposure.getInfo().setVisitInfo(visitInfo)
def _makeCamera(self, policy, repositoryDir)
def map_expIdInfo(self, dataId, write=False)
def _setAmpDetector(self, item, dataId, trimmed=True)
Exposure ID and number of bits used.
def _standardizeExposure(self, mapping, item, dataId, filter=True, trimmed=True, setVisitInfo=True)
def _extractDetectorName(self, dataId)
def _setFilter(self, mapping, item, dataId)
def _setCcdDetector(self, item, dataId, trimmed=True)
def bypass_defects(self, datasetType, pythonType, butlerLocation, dataId)
def std_bfKernel(self, item, dataId)
def getKeys(self, datasetType, level)
def map_defects(self, dataId, write=False)
def map_camera(self, dataId, write=False)
def std_raw(self, item, dataId)
def backup(self, datasetType, dataId)
def _setupRegistry(self, name, description, path, policy, policyKey, storage, searchParents=True, posixIfNoSql=True)
def map_skypolicy(self, dataId)
def std_skypolicy(self, item, dataId)
def _defectLookup(self, dataId)
def bypass_camera(self, datasetType, pythonType, butlerLocation, dataId)
def _initMappings(self, policy, rootStorage=None, calibStorage=None, provided=None)
def getDefaultSubLevel(self, level)
def _transformId(self, dataId)
def getDefaultLevel(self)
def __init__(self, policy, repositoryDir, root=None, registry=None, calibRoot=None, calibRegistry=None, provided=None, parentRegistry=None, repositoryCfg=None)
def bypass_expIdInfo(self, datasetType, pythonType, location, dataId)
def exposureFromImage(image, dataId=None, mapper=None, logger=None, setVisitInfo=True)
def getShortCcdName(ccdName)
def _getCcdKeyVal(self, dataId)
Utility functions.