Coverage for python/lsst/daf/butler/remote_butler/_remote_file_transfer_source.py: 47%

1# This file is part of daf_butler. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This software is dual licensed under the GNU General Public License and also 

10# under a 3-clause BSD license. Recipients may choose which of these licenses 

11# to use; please see the files gpl-3.0.txt and/or bsd_license.txt, 

12# respectively. If you choose the GPL option then the following text applies 

13# (but note that there is still no warranty even if you opt for BSD instead): 

14# 

15# This program is free software: you can redistribute it and/or modify 

16# it under the terms of the GNU General Public License as published by 

17# the Free Software Foundation, either version 3 of the License, or 

18# (at your option) any later version. 

19# 

20# This program is distributed in the hope that it will be useful, 

21# but WITHOUT ANY WARRANTY; without even the implied warranty of 

22# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

23# GNU General Public License for more details. 

24# 

25# You should have received a copy of the GNU General Public License 

26# along with this program. If not, see <http://www.gnu.org/licenses/>. 

27 

28from collections.abc import Callable, Iterable, Iterator 

29from contextlib import contextmanager 

30from typing import Any, cast 

31 

32from lsst.daf.butler._dataset_ref import DatasetId, DatasetRef 

33from lsst.resources import ResourcePath 

34from lsst.resources.http import HttpResourcePath 

35from lsst.utils.iteration import chunk_iterable 

36 

37from .._location import Location 

38from ..datastore import FileTransferMap, FileTransferRecord, FileTransferSource 

39from ..datastore.stored_file_info import StoredFileInfo 

40from ._get import convert_http_url_to_resource_path 

41from ._http_connection import RemoteButlerHttpConnection, parse_model 

42from .server_models import ( 

43 FileTransferRecordModel, 

44 GetFileTransferInfoRequestModel, 

45 GetFileTransferInfoResponseModel, 

46) 

47 

48 

49class RemoteFileTransferSource(FileTransferSource): 

50 """Implementation of `FileTransferSource` that retrieves information from 

51 Butler server. 

52 

53 Parameters 

54 ---------- 

55 connection : `RemoteButlerHttpConnection` 

56 HTTP connection used to access the Butler server. 

57 """ 

58 

59 def __init__(self, connection: RemoteButlerHttpConnection) -> None: 

60 self._connection = connection 

61 self.name = f"RemoteFileTransferSource{connection.server_url}" 

62 

63 def get_file_info_for_transfer(self, dataset_ids: Iterable[DatasetId]) -> FileTransferMap: 

64 output: FileTransferMap = {} 

65 for chunk in chunk_iterable(dataset_ids, GetFileTransferInfoRequestModel.MAX_ITEMS_PER_REQUEST): 

66 request = GetFileTransferInfoRequestModel(dataset_ids=chunk) 

67 response = self._connection.post("file_transfer", request) 

68 model = parse_model(response, GetFileTransferInfoResponseModel) 

69 for id, records in model.files.items(): 

70 output[id] = [self._deserialize_file_transfer_record(r) for r in records] 

71 

72 return output 

73 

74 def locate_missing_files_for_transfer( 

75 self, refs: Iterable[DatasetRef], artifact_existence: dict[ResourcePath, bool] 

76 ) -> FileTransferMap: 

77 # The server does not provide an alternate way to look up files that 

78 # could not be found using the file transfer endpoint. 

79 return {} 

80 

81 def _deserialize_file_transfer_record(self, record: FileTransferRecordModel) -> FileTransferRecord: 

82 resource_path = convert_http_url_to_resource_path(record.url, self._connection.auth, record.auth) 

83 resource_path = _tweak_uri_for_unit_test(resource_path) 

84 

85 return FileTransferRecord( 

86 location=Location(None, resource_path), 

87 file_info=StoredFileInfo.from_simple(record.file_info), 

88 ) 

89 

90 

91def _tweak_uri_for_unit_test(path: ResourcePath) -> ResourcePath: 

92 # Provide a place for unit tests to hook in and modify URLs, since there is 

93 # no actual HTTP server reachable via a domain name during testing. 

94 return path 

95 

96 

97@contextmanager 

98def mock_file_transfer_uris_for_unit_test( 

99 callback: Callable[[HttpResourcePath], HttpResourcePath], 

100) -> Iterator[None]: 

101 """Hooks into the RemoteButler file transfer logic to modify URLs for unit 

102 testing. The given callback will be used to transform file download URLs 

103 before attempting to access them. 

104 

105 Parameters 

106 ---------- 

107 callback : `~collections.abc.Callable` 

108 A function that takes an `HttpResourcePath` as its only parameter and 

109 returns a modified `HttpResourcePath`. 

110 """ 

111 global _tweak_uri_for_unit_test 

112 orig = _tweak_uri_for_unit_test 

113 _tweak_uri_for_unit_test = cast(Any, callback) 

114 try: 

115 yield 

116 finally: 

117 _tweak_uri_for_unit_test = orig