Coverage for python/lsst/analysis/tools/interfaces/datastore/_dispatcher.py: 13%

281 statements  

« prev     ^ index     » next       coverage.py v7.5.1, created at 2024-05-10 11:04 +0000

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("SasquatchDispatchPartialFailure", "SasquatchDispatchFailure", "SasquatchDispatcher") 

25 

26"""Sasquatch datastore""" 

27import calendar 

28import datetime 

29import json 

30import logging 

31import math 

32import re 

33from collections.abc import Mapping, MutableMapping, Sequence 

34from dataclasses import dataclass 

35from typing import TYPE_CHECKING, Any, cast 

36from uuid import UUID, uuid4 

37 

38import requests 

39from lsst.daf.butler import DatasetRef 

40from lsst.resources import ResourcePath 

41from lsst.utils.packages import getEnvironmentPackages 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .. import MetricMeasurementBundle 

45 

46 

47log = logging.getLogger(__name__) 

48 

49# Constants assocated with SasquatchDispatcher 

50PARTITIONS = 1 

51REPLICATION_FACTOR = 3 

52 

53IDENTIFIER_KEYS = [ 

54 "detector", 

55 "patch", 

56 "skymap", 

57 "visit", 

58 "tract", 

59 "physical_filter", 

60 "instrument", 

61 "band", 

62 "exposure", 

63 "group", 

64 "day_obs", 

65] 

66 

67 

68class SasquatchDispatchPartialFailure(RuntimeError): 

69 """This indicates that a Sasquatch dispatch was partially successful.""" 

70 

71 pass 

72 

73 

74class SasquatchDispatchFailure(RuntimeError): 

75 """This indicates that dispatching a 

76 `~lsst.analysis.tool.interface.MetricMeasurementBundle` failed. 

77 """ 

78 

79 pass 

80 

81 

82def _tag2VersionTime(productStr: str) -> tuple[str, float]: 

83 """Determine versions and dates from the string returned from 

84 getEnvironmentPackages. 

85 

86 The `~lsst.utils.packages.genEnvironmentPackages` function returns the 

87 setup version associated with a product, along with a list of tags that 

88 have been added to it. 

89 

90 This method splits up that return string, and determines the earliest date 

91 associated with the setup package version. 

92 

93 Parameters 

94 ---------- 

95 productStr : `str` 

96 The product string returned from a lookup on the result of a call to 

97 `~lsst.utils.packages.getEnvironmentPackages`. 

98 

99 Returns 

100 ------- 

101 result : `tuple` of `str`, `datetime.datetime` 

102 The first `str` is the version of the package, and the second is the 

103 datetime object associated with that released version. 

104 

105 Raises 

106 ------ 

107 ValueError 

108 Raised if there are no tags which correspond to dates. 

109 """ 

110 times: list[datetime.datetime] = [] 

111 version = productStr.split()[0] 

112 tags: str = re.findall("[(](.*)[)]", productStr)[0] 

113 for tag in tags.split(): 

114 numDots = tag.count(".") 

115 numUnder = tag.count("_") 

116 separator = "_" 

117 if numDots > numUnder: 

118 separator = "." 

119 match tag.split(separator): 

120 # Daily tag branch. 

121 case ("d", year, month, day): 

122 dt = datetime.datetime(year=int(year), month=int(month), day=int(day)) 

123 # Weekly tag branch. 

124 case ("w", year, week): 

125 iyear = int(year) 

126 iweek = int(week) 

127 # Use 4 as the day because releases are available starting 

128 # on Thursday 

129 dayOfWeek = 4 

130 

131 # Find the first week to contain a thursday in it 

132 cal = calendar.Calendar() 

133 cal.setfirstweekday(6) 

134 i = 0 

135 for i, iterWeek in enumerate(cal.monthdatescalendar(iyear, 1)): 

136 if iterWeek[dayOfWeek].month == 1: 

137 break 

138 # Handle fromisocalendar not being able to handle week 53 

139 # in the case were the date was going to subtract 7 days anyway 

140 if i and iweek == 53: 

141 i = 0 

142 iweek = 52 

143 delta = datetime.timedelta(days=7 * i) 

144 

145 # Correct for a weekly being issued in the last week of the 

146 # previous year, as Thursdays don't always line up evenly in 

147 # a week / year split. 

148 dt = datetime.datetime.fromisocalendar(iyear, iweek, dayOfWeek) - delta 

149 # Skip tags that can't be understood. 

150 case _: 

151 continue 

152 times.append(dt) 

153 if len(times) == 0: 

154 raise ValueError("Could not find any tags corresponding to dates") 

155 minTime = min(times) 

156 minTime.replace(tzinfo=datetime.timezone.utc) 

157 return version, minTime.timestamp() 

158 

159 

160@dataclass 

161class SasquatchDispatcher: 

162 """This class mediates the transfer of MetricMeasurementBundles to a 

163 Sasquatch http kafka proxy server. 

164 """ 

165 

166 url: str 

167 """Url of the Sasquatch proxy server""" 

168 

169 token: str 

170 """Authentication token used in communicating with the proxy server""" 

171 

172 namespace: str = "lsst.dm" 

173 """The namespace in Sasquatch in which to write the uploaded metrics""" 

174 

175 def __post_init__(self) -> None: 

176 match ResourcePath(self.url).scheme: 

177 case "http" | "https": 

178 pass 

179 case _: 

180 raise ValueError("Proxy server must be locatable with either http or https") 

181 

182 self._cluster_id: str | None = None 

183 

184 @property 

185 def clusterId(self) -> str: 

186 """ClusterId of the Kafka proxy 

187 

188 Notes 

189 ----- 

190 The cluster Id will be fetched with a network call if it is not 

191 already cached. 

192 """ 

193 if self._cluster_id is None: 

194 self._populateClusterId() 

195 return cast(str, self._cluster_id) 

196 

197 def _populateClusterId(self) -> None: 

198 """Get Sasquatch kafka cluster ID.""" 

199 

200 headers = {"content-type": "application/json"} 

201 r = requests.get(f"{self.url}/v3/clusters", headers=headers) 

202 

203 if r.status_code == requests.codes.ok: 

204 cluster_id = r.json()["data"][0]["cluster_id"] 

205 

206 self._cluster_id = str(cluster_id) 

207 else: 

208 log.error("Could not retrieve the cluster id for the specified url") 

209 raise SasquatchDispatchFailure("Could not retrieve the cluster id for the specified url") 

210 

211 def _create_topic(self, topic_name: str) -> bool: 

212 """Create a kafka topic in Sasquatch. 

213 

214 Parameters 

215 ---------- 

216 topic_name : `str` 

217 The name of the kafka topic to create 

218 

219 Returns 

220 ------- 

221 status : `bool` 

222 If this does not encounter an error it will return a True success 

223 code, else it will return a False code. 

224 

225 """ 

226 

227 headers = {"content-type": "application/json"} 

228 

229 topic_config = { 

230 "topic_name": f"{self.namespace}.{topic_name}", 

231 "partitions_count": PARTITIONS, 

232 "replication_factor": REPLICATION_FACTOR, 

233 } 

234 

235 r = requests.post( 

236 f"{self.url}/v3/clusters/{self.clusterId}/topics", json=topic_config, headers=headers 

237 ) 

238 

239 if r.status_code == requests.codes.created: 

240 log.debug("Created topic %s.%s", self.namespace, topic_name) 

241 return True 

242 elif r.status_code == requests.codes.bad_request: 

243 log.debug("Topic %s.%s already exists.", self.namespace, topic_name) 

244 return True 

245 else: 

246 log.error("Uknown error occured creating kafka topic %s %s", r.status_code, r.json()) 

247 return False 

248 

249 def _generateAvroSchema(self, metric: str, record: MutableMapping[str, Any]) -> tuple[str, bool]: 

250 """Infer the Avro schema from the record payload. 

251 

252 Parameters 

253 ---------- 

254 metric : `str` 

255 The name of the metric 

256 record : `MutableMapping` 

257 The prepared record for which a schema is to be generated 

258 

259 Returns 

260 ------- 

261 resultSchema : `str` 

262 A json encoded string of the resulting avro schema 

263 errorCode : bool 

264 A boolean indicating if any record fields had to be trimmed because 

265 a suitable schema could not be generated. True if records were 

266 removed, False otherwise. 

267 """ 

268 schema: dict[str, Any] = {"type": "record", "namespace": self.namespace, "name": metric} 

269 

270 # Record if any records needed to be trimmed 

271 resultsTrimmed = False 

272 

273 fields = list() 

274 # If avro schemas cant be generated for values, they should be removed 

275 # from the records. 

276 keysToRemove: list[str] = [] 

277 for key in record: 

278 value = record[key] 

279 avroType: Mapping[str, Any] 

280 if "timestamp" in key: 

281 avroType = {"type": "double"} 

282 else: 

283 avroType = self._python2Avro(value) 

284 if len(avroType) == 0: 

285 continue 

286 if avroType.get("error_in_conversion"): 

287 keysToRemove.append(key) 

288 resultsTrimmed = True 

289 continue 

290 fields.append({"name": key, **avroType}) 

291 

292 # remove any key that failed to have schema generated 

293 for key in keysToRemove: 

294 record.pop(key) 

295 

296 schema["fields"] = fields 

297 

298 return json.dumps(schema), resultsTrimmed 

299 

300 def _python2Avro(self, value: Any) -> Mapping: 

301 """Map python type to avro schema 

302 

303 Parameters 

304 ---------- 

305 value : `Any` 

306 Any python parameter. 

307 

308 Returns 

309 ------- 

310 result : `Mapping` 

311 Return a mapping that represents an entry in an avro schema. 

312 """ 

313 match value: 

314 case float() | None: 

315 return {"type": "float", "default": 0.0} 

316 case str(): 

317 return {"type": "string", "default": ""} 

318 case int(): 

319 return {"type": "int", "default": 0} 

320 case Sequence(): 

321 tmp = {self._python2Avro(item)["type"] for item in value} 

322 if len(tmp) == 0: 

323 return {} 

324 if len(tmp) > 1: 

325 log.error( 

326 "Sequence contains mixed types: %s, must be homogeneous for avro conversion " 

327 "skipping record", 

328 tmp, 

329 ) 

330 return {"error_in_conversion": True} 

331 return {"type": "array", "items": tmp.pop()} 

332 case _: 

333 log.error("Unsupported type %s, skipping record", type(value)) 

334 return {} 

335 

336 def _handleReferencePackage(self, meta: MutableMapping, bundle: MetricMeasurementBundle) -> None: 

337 """Check to see if there is a reference package. 

338 

339 if there is a reference package, determine the datetime associated with 

340 this reference package. Save the package, the version, and the date to 

341 the common metric fields. 

342 

343 Parameters 

344 ---------- 

345 meta : `MutableMapping` 

346 A mapping which corresponds to fields which should be encoded in 

347 all records. 

348 bundle : `MetricMeasurementBundle` 

349 The bundled metrics 

350 """ 

351 package_version, package_timestamp = "", 0.0 

352 if ref_package := getattr(bundle, "reference_package", ""): 

353 ref_package = bundle.reference_package 

354 packages = getEnvironmentPackages(True) 

355 if package_info := packages.get(ref_package): 

356 try: 

357 package_version, package_timestamp = _tag2VersionTime(package_info) 

358 except ValueError: 

359 # Could not extract package timestamp leaving empty 

360 pass 

361 # explicit handle if None was set in the bundle for the package 

362 meta["reference_package"] = ref_package or "" 

363 meta["reference_package_version"] = package_version 

364 meta["reference_package_timestamp"] = package_timestamp 

365 

366 def _handleTimes(self, meta: MutableMapping, bundle: MetricMeasurementBundle, run: str) -> None: 

367 """Add times to the meta fields mapping. 

368 

369 Add all appropriate timestamp fields to the meta field mapping. These 

370 will be added to all records. 

371 

372 This method will also look at the bundle to see if it defines a 

373 preferred time. It so it sets that time as the main time stamp to be 

374 used for this record. 

375 

376 Parameters 

377 ---------- 

378 meta : `MutableMapping` 

379 A mapping which corresponds to fields which should be encoded in 

380 all records. 

381 bundle : `MetricMeasurementBundle` 

382 The bundled metrics 

383 run : `str` 

384 The `~lsst.daf.butler.Butler` collection where the 

385 `MetricMeasurementBundle` is stored. 

386 """ 

387 # Determine the timestamp associated with the run, if someone abused 

388 # the run collection, use the current timestamp 

389 if re.match(r"\d{8}T\d{6}Z", stamp := run.split("/")[-1]): 

390 run_timestamp = datetime.datetime.strptime(stamp, r"%Y%m%dT%H%M%S%z") 

391 else: 

392 run_timestamp = datetime.datetime.now() 

393 meta["run_timestamp"] = run_timestamp.timestamp() 

394 

395 # If the bundle supports supplying timestamps, dispatch on the type 

396 # specified. 

397 if hasattr(bundle, "timestamp_version") and bundle.timestamp_version: 

398 match bundle.timestamp_version: 

399 case "reference_package_timestamp": 

400 if not meta["reference_package_timestamp"]: 

401 log.error("Reference package timestamp is empty, using run_timestamp") 

402 meta["timestamp"] = meta["run_timestamp"] 

403 else: 

404 meta["timestamp"] = meta["reference_package_timestamp"] 

405 case "run_timestamp": 

406 meta["timestamp"] = meta["run_timestamp"] 

407 case "current_timestamp": 

408 timeStamp = datetime.datetime.now() 

409 meta["timestamp"] = timeStamp.timestamp() 

410 case "dataset_timestamp": 

411 log.error("dataset timestamps are not yet supported, run_timestamp will be used") 

412 meta["timestamp"] = meta["run_timestamp"] 

413 case str(value) if "explicit_timestamp" in value: 

414 try: 

415 _, splitTime = value.split(":") 

416 except ValueError as excpt: 

417 raise ValueError( 

418 "Explicit timestamp must be given in the format 'explicit_timestamp:datetime', " 

419 "where datetime is given in the form '%Y%m%dT%H%M%S%z" 

420 ) from excpt 

421 meta["timestamp"] = datetime.datetime.strptime(splitTime, r"%Y%m%dT%H%M%S%z").timestamp() 

422 case _: 

423 log.error( 

424 "Timestamp version %s is not supported, run_timestamp will be used", 

425 bundle.timestamp_version, 

426 ) 

427 meta["timestamp"] = meta["run_timestamp"] 

428 # Default to using the run_timestamp. 

429 else: 

430 meta["timestamp"] = meta["run_timestamp"] 

431 

432 def _handleIdentifier( 

433 self, 

434 meta: MutableMapping, 

435 identifierFields: Mapping[str, Any] | None, 

436 datasetIdentifier: str | None, 

437 bundle: MetricMeasurementBundle, 

438 ) -> None: 

439 """Add an identifier to the meta record mapping. 

440 

441 If the bundle declares a dataset identifier to use add that to the 

442 record, otherwise use 'Generic' as the identifier. If the 

443 datasetIdentifier parameter is specified, that is used instead of 

444 anything specified by the bundle. 

445 

446 This will also add any identifier fields supplied to the meta record 

447 mapping. 

448 

449 Together these values (in addition to the timestamp and topic) should 

450 uniquely identify an upload to the Sasquatch system. 

451 

452 Parameters 

453 ---------- 

454 meta : `MutableMapping` 

455 A mapping which corresponds to fields which should be encoded in 

456 all records. 

457 identifierFields: `Mapping` or `None` 

458 The keys and values in this mapping will be both added as fields 

459 in the record, and used in creating a unique tag for the uploaded 

460 dataset type. I.e. the timestamp, and the tag will be unique, and 

461 each record will belong to one combination of such. 

462 datasetIdentifier : `str` or `None` 

463 A string which will be used in creating unique identifier tags. 

464 bundle : `MetricMeasurementBundle` 

465 The bundle containing metric values to upload. 

466 """ 

467 identifier: str 

468 if datasetIdentifier is not None: 

469 identifier = datasetIdentifier 

470 elif hasattr(bundle, "dataset_identifier") and bundle.dataset_identifier is not None: 

471 identifier = bundle.dataset_identifier 

472 else: 

473 identifier = "Generic" 

474 

475 meta["dataset_tag"] = identifier 

476 

477 if identifierFields is None: 

478 identifierFields = {} 

479 for key in IDENTIFIER_KEYS: 

480 value = identifierFields.get(key, "") 

481 meta[key] = f"{value}" 

482 

483 def _prepareBundle( 

484 self, 

485 bundle: MetricMeasurementBundle, 

486 run: str, 

487 datasetType: str, 

488 timestamp: datetime.datetime | None = None, 

489 id: UUID | None = None, 

490 identifierFields: Mapping | None = None, 

491 datasetIdentifier: str | None = None, 

492 extraFields: Mapping | None = None, 

493 ) -> tuple[Mapping[str, list[Any]], bool]: 

494 """Encode all of the inputs into a format that can be sent to the 

495 kafka proxy server. 

496 

497 Parameters 

498 ---------- 

499 bundle : `MetricMeasurementBundle` 

500 The bundle containing metric values to upload. 

501 run : `str` 

502 The run name to associate with these metric values. If this bundle 

503 is also stored in the butler, this should be the butler run 

504 collection the bundle is stored in the butler. 

505 datasetType : `str` 

506 The dataset type name associated with this 

507 `MetricMeasurementBundle` 

508 timestamp : `datetime.datetime`, optional 

509 The timestamp to be associated with the measurements in the ingress 

510 database. If this value is None, timestamp will be set by the run 

511 time or current time. 

512 id : `UUID`, optional 

513 The UUID of the `MetricMeasurementBundle` within the butler. If 

514 `None`, a new random UUID will be generated so that each record in 

515 Sasquatch will have a unique value. 

516 identifierFields: `Mapping`, optional 

517 The keys and values in this mapping will be both added as fields 

518 in the record, and used in creating a unique tag for the uploaded 

519 dataset type. I.e. the timestamp, and the tag will be unique, and 

520 each record will belong to one combination of such. 

521 datasetIdentifier : `str`, optional 

522 A string which will be used in creating unique identifier tags. 

523 extraFields: `Mapping`, optional 

524 Extra mapping keys and values that will be added as fields to the 

525 dispatched record. 

526 

527 Returns 

528 ------- 

529 result : `Mapping` of `str` to `list` 

530 A mapping of metric name of list of metric measurement records. 

531 status : `bool` 

532 A status boolean indicating if some records had to be skipped due 

533 to a problem parsing the bundle. 

534 """ 

535 if id is None: 

536 id = uuid4() 

537 sid = str(id) 

538 meta: dict[str, Any] = dict() 

539 

540 # Add other associated common fields 

541 meta["id"] = sid 

542 meta["run"] = run 

543 meta["dataset_type"] = datasetType 

544 

545 # Check to see if the bundle declares a reference package 

546 self._handleReferencePackage(meta, bundle) 

547 

548 # Handle the various timestamps that could be associated with a record 

549 self._handleTimes(meta, bundle, run) 

550 

551 # Always use the supplied timestamp if one was passed to use. 

552 if timestamp is not None: 

553 meta["timestamp"] = timestamp.timestamp() 

554 

555 self._handleIdentifier(meta, identifierFields, datasetIdentifier, bundle) 

556 

557 # Add in any other fields that were supplied to the function call. 

558 if extraFields is not None: 

559 meta.update(extraFields) 

560 

561 metricRecords: dict[str, list[Any]] = dict() 

562 

563 # Record if any records needed skipped 

564 resultsTrimmed = False 

565 

566 # Look at each of the metrics in the bundle (name, values) 

567 for metric, measurements in bundle.items(): 

568 # Create a list which will contain the records for each measurement 

569 # associated with metric. 

570 metricRecordList = metricRecords.setdefault(metric, list()) 

571 

572 record: dict[str, Any] = meta.copy() 

573 

574 # loop over each metric measurement within the metric 

575 for measurement in measurements: 

576 # need to extract any tags, package info, etc 

577 note_key = f"{measurement.metric_name.metric}.metric_tags" 

578 record["tags"] = dict(measurement.notes.items()).get(note_key, list()) 

579 

580 # Missing values are replaced by 0 in sasquatch, see RFC-763. 

581 name = "" 

582 value = 0.0 

583 match measurement.json: 

584 case {"metric": name, "value": None}: 

585 pass 

586 case {"metric": name, "value": value}: 

587 if math.isnan(value): 

588 log.error( 

589 "Measurement %s had a value that is a NaN, dispatch will be skipped", 

590 measurement, 

591 ) 

592 resultsTrimmed = True 

593 continue 

594 pass 

595 case {"value": _}: 

596 log.error("Measurement %s does not contain the key 'metric'", measurement) 

597 resultsTrimmed = True 

598 continue 

599 case {"metric": _}: 

600 log.error("Measurement %s does not contain the key 'value'", measurement) 

601 resultsTrimmed = True 

602 continue 

603 record[name] = value 

604 

605 metricRecordList.append({"value": record}) 

606 return metricRecords, resultsTrimmed 

607 

608 def dispatch( 

609 self, 

610 bundle: MetricMeasurementBundle, 

611 run: str, 

612 datasetType: str, 

613 timestamp: datetime.datetime | None = None, 

614 id: UUID | None = None, 

615 datasetIdentifier: str | None = None, 

616 identifierFields: Mapping | None = None, 

617 extraFields: Mapping | None = None, 

618 ) -> None: 

619 """Dispatch a `MetricMeasurementBundle` to Sasquatch. 

620 

621 Parameters 

622 ---------- 

623 bundle : `MetricMeasurementBundle` 

624 The bundle containing metric values to upload. 

625 run : `str` 

626 The run name to associate with these metric values. If this bundle 

627 is also stored in the butler, this should be the butler run 

628 collection the bundle is stored in the butler. This will be used 

629 in generating uniqueness constraints in Sasquatch. 

630 datasetType : `str` 

631 The dataset type name associated with this 

632 `MetricMeasurementBundle`. 

633 timestamp : `datetime.datetime`, optional 

634 The timestamp to be associated with the measurements in the ingress 

635 database. If this value is None, timestamp will be set by the run 

636 time or current time. 

637 id : `UUID`, optional 

638 The UUID of the `MetricMeasurementBundle` within the Butler. If 

639 `None`, a new random UUID will be generated so that each record in 

640 Sasquatch will have a unique value. 

641 datasetIdentifier : `str`, optional 

642 A string which will be used in creating unique identifier tags. If 

643 `None`, a default value will be inserted. 

644 identifierFields: `Mapping`, optional 

645 The keys and values in this mapping will be both added as fields 

646 in the record, and used in creating a unique tag for the uploaded 

647 dataset type. I.e. the timestamp, and the tag will be unique, and 

648 each record will belong to one combination of such. Examples of 

649 entries would be things like visit or tract. 

650 extraFields: `Mapping`, optional 

651 Extra mapping keys and values that will be added as fields to the 

652 dispatched record. 

653 

654 Raises 

655 ------ 

656 SasquatchDispatchPartialFailure 

657 Raised if there were any errors in dispatching a bundle. 

658 """ 

659 if id is None: 

660 id = uuid4() 

661 

662 # Prepare the bundle by transforming it to a list of metric records 

663 metricRecords, recordsTrimmed = self._prepareBundle( 

664 bundle=bundle, 

665 run=run, 

666 datasetType=datasetType, 

667 timestamp=timestamp, 

668 id=id, 

669 datasetIdentifier=datasetIdentifier, 

670 identifierFields=identifierFields, 

671 extraFields=extraFields, 

672 ) 

673 

674 headers = {"content-type": "application/vnd.kafka.avro.v2+json"} 

675 data: dict[str, Any] = dict() 

676 partialUpload = False 

677 uploadFailed = [] 

678 

679 for metric, record in metricRecords.items(): 

680 # create the kafka topic if it does not already exist 

681 if not self._create_topic(metric): 

682 log.error("Topic not created, skipping dispatch of %s", metric) 

683 continue 

684 recordValue = record[0]["value"] 

685 # Generate schemas for each record 

686 data["value_schema"], schemaTrimmed = self._generateAvroSchema(metric, recordValue) 

687 data["records"] = record 

688 

689 if schemaTrimmed: 

690 partialUpload = True 

691 

692 r = requests.post(f"{self.url}/topics/{self.namespace}.{metric}", json=data, headers=headers) 

693 

694 if r.status_code == requests.codes.ok: 

695 log.debug("Succesfully sent data for metric %s", metric) 

696 uploadFailed.append(False) 

697 else: 

698 log.error( 

699 "There was a problem submitting the metric %s: %s, %s", metric, r.status_code, r.json() 

700 ) 

701 uploadFailed.append(True) 

702 partialUpload = True 

703 

704 # There may be no metrics to try to upload, and thus the uploadFailed 

705 # list may be empty, check before issuing failure 

706 if len(uploadFailed) > 0 and all(uploadFailed): 

707 raise SasquatchDispatchFailure("All records were unable to be uploaded.") 

708 

709 if partialUpload or recordsTrimmed: 

710 raise SasquatchDispatchPartialFailure("One or more records may not have been uploaded entirely") 

711 

712 def dispatchRef( 

713 self, 

714 bundle: MetricMeasurementBundle, 

715 ref: DatasetRef, 

716 timestamp: datetime.datetime | None = None, 

717 extraFields: Mapping | None = None, 

718 datasetIdentifier: str | None = None, 

719 ) -> None: 

720 """Dispatch a `MetricMeasurementBundle` to Sasquatch with a known 

721 `DatasetRef`. 

722 

723 Parameters 

724 ---------- 

725 bundle : `MetricMeasurementBundle` 

726 The bundle containing metric values to upload. 

727 ref : `DatasetRef` 

728 The `Butler` dataset ref corresponding to the input 

729 `MetricMeasurementBundle`. 

730 timestamp : `datetime.datetime`, optional 

731 The timestamp to be associated with the measurements in the ingress 

732 database. If this value is None, timestamp will be set by the run 

733 time or current time. 

734 extraFields: `Mapping`, optional 

735 Extra mapping keys and values that will be added as fields to the 

736 dispatched record if not None. 

737 datasetIdentifier : `str`, optional 

738 A string which will be used in creating unique identifier tags. If 

739 None, a default value will be inserted. 

740 

741 Raises 

742 ------ 

743 SasquatchDispatchPartialFailure 

744 Raised if there were any errors in dispatching a bundle. 

745 """ 

746 # Parse the relevant info out of the dataset ref. 

747 serializedRef = ref.to_simple() 

748 id = serializedRef.id 

749 if serializedRef.run is None: 

750 run = "<unknown>" 

751 else: 

752 run = serializedRef.run 

753 dstype = serializedRef.datasetType 

754 datasetType = dstype.name if dstype is not None else "" 

755 dataRefMapping = serializedRef.dataId.dataId if serializedRef.dataId else None 

756 

757 self.dispatch( 

758 bundle, 

759 run=run, 

760 timestamp=timestamp, 

761 datasetType=datasetType, 

762 id=id, 

763 identifierFields=dataRefMapping, 

764 extraFields=extraFields, 

765 datasetIdentifier=datasetIdentifier, 

766 )