Coverage for python / lsst / daf / butler / _rubin / file_datasets.py: 37%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from __future__ import annotations 

29 

30__all__ = ("transfer_datasets_to_datastore",) 

31 

32from collections.abc import Iterable 

33 

34from .._butler import Butler 

35from .._butler_config import ButlerConfig 

36from .._dataset_ref import DatasetRef 

37from .._file_dataset import FileDataset 

38from .._standalone_datastore import instantiate_standalone_datastore 

39from ..direct_butler import DirectButler 

40 

41 

42def transfer_datasets_to_datastore( 

43 source_butler: Butler, output_config: ButlerConfig, refs: Iterable[DatasetRef] 

44) -> list[FileDataset]: 

45 """Copy datasets into a `Datastore` without writing entries into the 

46 associated registry database. 

47 

48 Parameters 

49 ---------- 

50 source_butler : `Butler` 

51 Butler instance from which to copy datasets. 

52 output_config : `ButlerConfig` 

53 Butler configuration specifying the configuration for the target 

54 datastore. 

55 refs : `~collections.abc.Iterable` [ `DatasetRef` ] 

56 List of datasets to copy into the target datastore. 

57 

58 Returns 

59 ------- 

60 file_datasets : `list` [ `FileDataset` ] 

61 `FileDataset` instances representing the artifacts copied to 

62 the target datastore. 

63 """ 

64 refs = list(refs) 

65 if len(refs) == 0: 

66 return [] 

67 

68 if not isinstance(source_butler, DirectButler): 

69 raise ValueError("Butler must be an instance of DirectButler") 

70 

71 # Refs must contain dimension records to allow for filename template 

72 # expansion in the datastore. 

73 refs = source_butler._registry.expand_refs(list(refs)) 

74 # Read out the absolute URLs of the datasets we are about to transfer. 

75 datasets = list(source_butler._datastore.export(refs, directory=None, transfer="direct")) 

76 

77 # Set up a temporary, initially empty, in-memory DB for the target 

78 # Datastore. 

79 dimension_universe = datasets[0].refs[0].dimensions.universe 

80 datastore, db = instantiate_standalone_datastore(output_config, dimension_universe) 

81 try: 

82 # Write the files to their destination in the target datastore. 

83 datastore.ingest(*datasets, transfer="copy", record_validation_info=False) 

84 # Read out the relative URLs of the files, for their location 

85 # in the target datastore. 

86 return list(datastore.export(refs, directory=None, transfer=None)) 

87 finally: 

88 db.dispose()