Coverage for python/lsst/analysis/tools/interfaces/datastore/_dispatcher.py: 13%

281 statements  

« prev     ^ index     » next       coverage.py v7.4.0, created at 2024-01-20 13:17 +0000

1# This file is part of analysis_tools. 

2# 

3# Developed for the LSST Data Management System. 

4# This product includes software developed by the LSST Project 

5# (http://www.lsst.org). 

6# See the COPYRIGHT file at the top-level directory of this distribution 

7# for details of code ownership. 

8# 

9# This program is free software: you can redistribute it and/or modify 

10# it under the terms of the GNU General Public License as published by 

11# the Free Software Foundation, either version 3 of the License, or 

12# (at your option) any later version. 

13# 

14# This program is distributed in the hope that it will be useful, 

15# but WITHOUT ANY WARRANTY; without even the implied warranty of 

16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the 

17# GNU General Public License for more details. 

18# 

19# You should have received a copy of the GNU General Public License 

20# along with this program. If not, see <http://www.gnu.org/licenses/>. 

21 

22from __future__ import annotations 

23 

24__all__ = ("SasquatchDispatchPartialFailure", "SasquatchDispatchFailure", "SasquatchDispatcher") 

25 

26"""Sasquatch datastore""" 

27import calendar 

28import datetime 

29import json 

30import logging 

31import math 

32import re 

33from collections.abc import Mapping, MutableMapping, Sequence 

34from dataclasses import dataclass 

35from typing import TYPE_CHECKING, Any, cast 

36from uuid import UUID, uuid4 

37 

38import requests 

39from lsst.daf.butler import DatasetRef 

40from lsst.resources import ResourcePath 

41from lsst.utils.packages import getEnvironmentPackages 

42 

43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true

44 from .. import MetricMeasurementBundle 

45 

46 

47log = logging.getLogger(__name__) 

48 

49# Constants assocated with SasquatchDispatcher 

50PARTITIONS = 1 

51REPLICATION_FACTOR = 3 

52 

53IDENTIFIER_KEYS = [ 

54 "detector", 

55 "patch", 

56 "skymap", 

57 "visit", 

58 "tract", 

59 "physical_filter", 

60 "instrument", 

61 "band", 

62 "exposure", 

63] 

64 

65 

66class SasquatchDispatchPartialFailure(RuntimeError): 

67 """This indicates that a Sasquatch dispatch was partially successful.""" 

68 

69 pass 

70 

71 

72class SasquatchDispatchFailure(RuntimeError): 

73 """This indicates that dispatching a 

74 `~lsst.analysis.tool.interface.MetricMeasurementBundle` failed. 

75 """ 

76 

77 pass 

78 

79 

80def _tag2VersionTime(productStr: str) -> tuple[str, float]: 

81 """Determine versions and dates from the string returned from 

82 getEnvironmentPackages. 

83 

84 The `~lsst.utils.packages.genEnvironmentPackages` function returns the 

85 setup version associated with a product, along with a list of tags that 

86 have been added to it. 

87 

88 This method splits up that return string, and determines the earliest date 

89 associated with the setup package version. 

90 

91 Parameters 

92 ---------- 

93 productStr : `str` 

94 The product string returned from a lookup on the result of a call to 

95 `~lsst.utils.packages.getEnvironmentPackages`. 

96 

97 Returns 

98 ------- 

99 result : `tuple` of `str`, `datetime.datetime` 

100 The first `str` is the version of the package, and the second is the 

101 datetime object associated with that released version. 

102 

103 Raises 

104 ------ 

105 ValueError 

106 Raised if there are no tags which correspond to dates. 

107 """ 

108 times: list[datetime.datetime] = [] 

109 version = productStr.split()[0] 

110 tags: str = re.findall("[(](.*)[)]", productStr)[0] 

111 for tag in tags.split(): 

112 numDots = tag.count(".") 

113 numUnder = tag.count("_") 

114 separator = "_" 

115 if numDots > numUnder: 

116 separator = "." 

117 match tag.split(separator): 

118 # Daily tag branch. 

119 case ("d", year, month, day): 

120 dt = datetime.datetime(year=int(year), month=int(month), day=int(day)) 

121 # Weekly tag branch. 

122 case ("w", year, week): 

123 iyear = int(year) 

124 iweek = int(week) 

125 # Use 4 as the day because releases are available starting 

126 # on Thursday 

127 dayOfWeek = 4 

128 

129 # Find the first week to contain a thursday in it 

130 cal = calendar.Calendar() 

131 cal.setfirstweekday(6) 

132 i = 0 

133 for i, iterWeek in enumerate(cal.monthdatescalendar(iyear, 1)): 

134 if iterWeek[dayOfWeek].month == 1: 

135 break 

136 # Handle fromisocalendar not being able to handle week 53 

137 # in the case were the date was going to subtract 7 days anyway 

138 if i and iweek == 53: 

139 i = 0 

140 iweek = 52 

141 delta = datetime.timedelta(days=7 * i) 

142 

143 # Correct for a weekly being issued in the last week of the 

144 # previous year, as Thursdays don't always line up evenly in 

145 # a week / year split. 

146 dt = datetime.datetime.fromisocalendar(iyear, iweek, dayOfWeek) - delta 

147 # Skip tags that can't be understood. 

148 case _: 

149 continue 

150 times.append(dt) 

151 if len(times) == 0: 

152 raise ValueError("Could not find any tags corresponding to dates") 

153 minTime = min(times) 

154 minTime.replace(tzinfo=datetime.timezone.utc) 

155 return version, minTime.timestamp() 

156 

157 

158@dataclass 

159class SasquatchDispatcher: 

160 """This class mediates the transfer of MetricMeasurementBundles to a 

161 Sasquatch http kafka proxy server. 

162 """ 

163 

164 url: str 

165 """Url of the Sasquatch proxy server""" 

166 

167 token: str 

168 """Authentication token used in communicating with the proxy server""" 

169 

170 namespace: str = "lsst.dm" 

171 """The namespace in Sasquatch in which to write the uploaded metrics""" 

172 

173 def __post_init__(self) -> None: 

174 match ResourcePath(self.url).scheme: 

175 case "http" | "https": 

176 pass 

177 case _: 

178 raise ValueError("Proxy server must be locatable with either http or https") 

179 

180 self._cluster_id: str | None = None 

181 

182 @property 

183 def clusterId(self) -> str: 

184 """ClusterId of the Kafka proxy 

185 

186 Notes 

187 ----- 

188 The cluster Id will be fetched with a network call if it is not 

189 already cached. 

190 """ 

191 if self._cluster_id is None: 

192 self._populateClusterId() 

193 return cast(str, self._cluster_id) 

194 

195 def _populateClusterId(self) -> None: 

196 """Get Sasquatch kafka cluster ID.""" 

197 

198 headers = {"content-type": "application/json"} 

199 r = requests.get(f"{self.url}/v3/clusters", headers=headers) 

200 

201 if r.status_code == requests.codes.ok: 

202 cluster_id = r.json()["data"][0]["cluster_id"] 

203 

204 self._cluster_id = str(cluster_id) 

205 else: 

206 log.error("Could not retrieve the cluster id for the specified url") 

207 raise SasquatchDispatchFailure("Could not retrieve the cluster id for the specified url") 

208 

209 def _create_topic(self, topic_name: str) -> bool: 

210 """Create a kafka topic in Sasquatch. 

211 

212 Parameters 

213 ---------- 

214 topic_name : `str` 

215 The name of the kafka topic to create 

216 

217 Returns 

218 ------- 

219 status : `bool` 

220 If this does not encounter an error it will return a True success 

221 code, else it will return a False code. 

222 

223 """ 

224 

225 headers = {"content-type": "application/json"} 

226 

227 topic_config = { 

228 "topic_name": f"{self.namespace}.{topic_name}", 

229 "partitions_count": PARTITIONS, 

230 "replication_factor": REPLICATION_FACTOR, 

231 } 

232 

233 r = requests.post( 

234 f"{self.url}/v3/clusters/{self.clusterId}/topics", json=topic_config, headers=headers 

235 ) 

236 

237 if r.status_code == requests.codes.created: 

238 log.debug("Created topic %s.%s", self.namespace, topic_name) 

239 return True 

240 elif r.status_code == requests.codes.bad_request: 

241 log.debug("Topic %s.%s already exists.", self.namespace, topic_name) 

242 return True 

243 else: 

244 log.error("Uknown error occured creating kafka topic %s %s", r.status_code, r.json()) 

245 return False 

246 

247 def _generateAvroSchema(self, metric: str, record: MutableMapping[str, Any]) -> tuple[str, bool]: 

248 """Infer the Avro schema from the record payload. 

249 

250 Parameters 

251 ---------- 

252 metric : `str` 

253 The name of the metric 

254 record : `MutableMapping` 

255 The prepared record for which a schema is to be generated 

256 

257 Returns 

258 ------- 

259 resultSchema : `str` 

260 A json encoded string of the resulting avro schema 

261 errorCode : bool 

262 A boolean indicating if any record fields had to be trimmed because 

263 a suitable schema could not be generated. True if records were 

264 removed, False otherwise. 

265 """ 

266 schema: dict[str, Any] = {"type": "record", "namespace": self.namespace, "name": metric} 

267 

268 # Record if any records needed to be trimmed 

269 resultsTrimmed = False 

270 

271 fields = list() 

272 # If avro schemas cant be generated for values, they should be removed 

273 # from the records. 

274 keysToRemove: list[str] = [] 

275 for key in record: 

276 value = record[key] 

277 avroType: Mapping[str, Any] 

278 if "timestamp" in key: 

279 avroType = {"type": "double"} 

280 else: 

281 avroType = self._python2Avro(value) 

282 if len(avroType) == 0: 

283 continue 

284 if avroType.get("error_in_conversion"): 

285 keysToRemove.append(key) 

286 resultsTrimmed = True 

287 continue 

288 fields.append({"name": key, **avroType}) 

289 

290 # remove any key that failed to have schema generated 

291 for key in keysToRemove: 

292 record.pop(key) 

293 

294 schema["fields"] = fields 

295 

296 return json.dumps(schema), resultsTrimmed 

297 

298 def _python2Avro(self, value: Any) -> Mapping: 

299 """Map python type to avro schema 

300 

301 Parameters 

302 ---------- 

303 value : `Any` 

304 Any python parameter. 

305 

306 Returns 

307 ------- 

308 result : `Mapping` 

309 Return a mapping that represents an entry in an avro schema. 

310 """ 

311 match value: 

312 case float() | None: 

313 return {"type": "float", "default": 0.0} 

314 case str(): 

315 return {"type": "string", "default": ""} 

316 case int(): 

317 return {"type": "int", "default": 0} 

318 case Sequence(): 

319 tmp = {self._python2Avro(item)["type"] for item in value} 

320 if len(tmp) == 0: 

321 return {} 

322 if len(tmp) > 1: 

323 log.error( 

324 "Sequence contains mixed types: %s, must be homogeneous for avro conversion " 

325 "skipping record", 

326 tmp, 

327 ) 

328 return {"error_in_conversion": True} 

329 return {"type": "array", "items": tmp.pop()} 

330 case _: 

331 log.error("Unsupported type %s, skipping record", type(value)) 

332 return {} 

333 

334 def _handleReferencePackage(self, meta: MutableMapping, bundle: MetricMeasurementBundle) -> None: 

335 """Check to see if there is a reference package. 

336 

337 if there is a reference package, determine the datetime associated with 

338 this reference package. Save the package, the version, and the date to 

339 the common metric fields. 

340 

341 Parameters 

342 ---------- 

343 meta : `MutableMapping` 

344 A mapping which corresponds to fields which should be encoded in 

345 all records. 

346 bundle : `MetricMeasurementBundle` 

347 The bundled metrics 

348 """ 

349 package_version, package_timestamp = "", 0.0 

350 if ref_package := getattr(bundle, "reference_package", ""): 

351 ref_package = bundle.reference_package 

352 packages = getEnvironmentPackages(True) 

353 if package_info := packages.get(ref_package): 

354 try: 

355 package_version, package_timestamp = _tag2VersionTime(package_info) 

356 except ValueError: 

357 # Could not extract package timestamp leaving empty 

358 pass 

359 # explicit handle if None was set in the bundle for the package 

360 meta["reference_package"] = ref_package or "" 

361 meta["reference_package_version"] = package_version 

362 meta["reference_package_timestamp"] = package_timestamp 

363 

364 def _handleTimes(self, meta: MutableMapping, bundle: MetricMeasurementBundle, run: str) -> None: 

365 """Add times to the meta fields mapping. 

366 

367 Add all appropriate timestamp fields to the meta field mapping. These 

368 will be added to all records. 

369 

370 This method will also look at the bundle to see if it defines a 

371 preferred time. It so it sets that time as the main time stamp to be 

372 used for this record. 

373 

374 Parameters 

375 ---------- 

376 meta : `MutableMapping` 

377 A mapping which corresponds to fields which should be encoded in 

378 all records. 

379 bundle : `MetricMeasurementBundle` 

380 The bundled metrics 

381 run : `str` 

382 The `~lsst.daf.butler.Butler` collection where the 

383 `MetricMeasurementBundle` is stored. 

384 """ 

385 # Determine the timestamp associated with the run, if someone abused 

386 # the run collection, use the current timestamp 

387 if re.match(r"\d{8}T\d{6}Z", stamp := run.split("/")[-1]): 

388 run_timestamp = datetime.datetime.strptime(stamp, r"%Y%m%dT%H%M%S%z") 

389 else: 

390 run_timestamp = datetime.datetime.now() 

391 meta["run_timestamp"] = run_timestamp.timestamp() 

392 

393 # If the bundle supports supplying timestamps, dispatch on the type 

394 # specified. 

395 if hasattr(bundle, "timestamp_version") and bundle.timestamp_version: 

396 match bundle.timestamp_version: 

397 case "reference_package_timestamp": 

398 if not meta["reference_package_timestamp"]: 

399 log.error("Reference package timestamp is empty, using run_timestamp") 

400 meta["timestamp"] = meta["run_timestamp"] 

401 else: 

402 meta["timestamp"] = meta["reference_package_timestamp"] 

403 case "run_timestamp": 

404 meta["timestamp"] = meta["run_timestamp"] 

405 case "current_timestamp": 

406 timeStamp = datetime.datetime.now() 

407 meta["timestamp"] = timeStamp.timestamp() 

408 case "dataset_timestamp": 

409 log.error("dataset timestamps are not yet supported, run_timestamp will be used") 

410 meta["timestamp"] = meta["run_timestamp"] 

411 case str(value) if "explicit_timestamp" in value: 

412 try: 

413 _, splitTime = value.split(":") 

414 except ValueError as excpt: 

415 raise ValueError( 

416 "Explicit timestamp must be given in the format 'explicit_timestamp:datetime', " 

417 "where datetime is given in the form '%Y%m%dT%H%M%S%z" 

418 ) from excpt 

419 meta["timestamp"] = datetime.datetime.strptime(splitTime, r"%Y%m%dT%H%M%S%z").timestamp() 

420 case _: 

421 log.error( 

422 "Timestamp version %s is not supported, run_timestamp will be used", 

423 bundle.timestamp_version, 

424 ) 

425 meta["timestamp"] = meta["run_timestamp"] 

426 # Default to using the run_timestamp. 

427 else: 

428 meta["timestamp"] = meta["run_timestamp"] 

429 

430 def _handleIdentifier( 

431 self, 

432 meta: MutableMapping, 

433 identifierFields: Mapping[str, Any] | None, 

434 datasetIdentifier: str | None, 

435 bundle: MetricMeasurementBundle, 

436 ) -> None: 

437 """Add an identifier to the meta record mapping. 

438 

439 If the bundle declares a dataset identifier to use add that to the 

440 record, otherwise use 'Generic' as the identifier. If the 

441 datasetIdentifier parameter is specified, that is used instead of 

442 anything specified by the bundle. 

443 

444 This will also add any identifier fields supplied to the meta record 

445 mapping. 

446 

447 Together these values (in addition to the timestamp and topic) should 

448 uniquely identify an upload to the Sasquatch system. 

449 

450 Parameters 

451 ---------- 

452 meta : `MutableMapping` 

453 A mapping which corresponds to fields which should be encoded in 

454 all records. 

455 identifierFields: `Mapping` or `None` 

456 The keys and values in this mapping will be both added as fields 

457 in the record, and used in creating a unique tag for the uploaded 

458 dataset type. I.e. the timestamp, and the tag will be unique, and 

459 each record will belong to one combination of such. 

460 datasetIdentifier : `str` or `None` 

461 A string which will be used in creating unique identifier tags. 

462 bundle : `MetricMeasurementBundle` 

463 The bundle containing metric values to upload. 

464 """ 

465 identifier: str 

466 if datasetIdentifier is not None: 

467 identifier = datasetIdentifier 

468 elif hasattr(bundle, "dataset_identifier") and bundle.dataset_identifier is not None: 

469 identifier = bundle.dataset_identifier 

470 else: 

471 identifier = "Generic" 

472 

473 meta["dataset_tag"] = identifier 

474 

475 if identifierFields is None: 

476 identifierFields = {} 

477 for key in IDENTIFIER_KEYS: 

478 value = identifierFields.get(key, "") 

479 meta[key] = f"{value}" 

480 

481 def _prepareBundle( 

482 self, 

483 bundle: MetricMeasurementBundle, 

484 run: str, 

485 datasetType: str, 

486 timestamp: datetime.datetime | None = None, 

487 id: UUID | None = None, 

488 identifierFields: Mapping | None = None, 

489 datasetIdentifier: str | None = None, 

490 extraFields: Mapping | None = None, 

491 ) -> tuple[Mapping[str, list[Any]], bool]: 

492 """Encode all of the inputs into a format that can be sent to the 

493 kafka proxy server. 

494 

495 Parameters 

496 ---------- 

497 bundle : `MetricMeasurementBundle` 

498 The bundle containing metric values to upload. 

499 run : `str` 

500 The run name to associate with these metric values. If this bundle 

501 is also stored in the butler, this should be the butler run 

502 collection the bundle is stored in the butler. 

503 datasetType : `str` 

504 The dataset type name associated with this 

505 `MetricMeasurementBundle` 

506 timestamp : `datetime.datetime`, optional 

507 The timestamp to be associated with the measurements in the ingress 

508 database. If this value is None, timestamp will be set by the run 

509 time or current time. 

510 id : `UUID`, optional 

511 The UUID of the `MetricMeasurementBundle` within the butler. If 

512 `None`, a new random UUID will be generated so that each record in 

513 Sasquatch will have a unique value. 

514 identifierFields: `Mapping`, optional 

515 The keys and values in this mapping will be both added as fields 

516 in the record, and used in creating a unique tag for the uploaded 

517 dataset type. I.e. the timestamp, and the tag will be unique, and 

518 each record will belong to one combination of such. 

519 datasetIdentifier : `str`, optional 

520 A string which will be used in creating unique identifier tags. 

521 extraFields: `Mapping`, optional 

522 Extra mapping keys and values that will be added as fields to the 

523 dispatched record. 

524 

525 Returns 

526 ------- 

527 result : `Mapping` of `str` to `list` 

528 A mapping of metric name of list of metric measurement records. 

529 status : `bool` 

530 A status boolean indicating if some records had to be skipped due 

531 to a problem parsing the bundle. 

532 """ 

533 if id is None: 

534 id = uuid4() 

535 sid = str(id) 

536 meta: dict[str, Any] = dict() 

537 

538 # Add other associated common fields 

539 meta["id"] = sid 

540 meta["run"] = run 

541 meta["dataset_type"] = datasetType 

542 

543 # Check to see if the bundle declares a reference package 

544 self._handleReferencePackage(meta, bundle) 

545 

546 # Handle the various timestamps that could be associated with a record 

547 self._handleTimes(meta, bundle, run) 

548 

549 # Always use the supplied timestamp if one was passed to use. 

550 if timestamp is not None: 

551 meta["timestamp"] = timestamp.timestamp() 

552 

553 self._handleIdentifier(meta, identifierFields, datasetIdentifier, bundle) 

554 

555 # Add in any other fields that were supplied to the function call. 

556 if extraFields is not None: 

557 meta.update(extraFields) 

558 

559 metricRecords: dict[str, list[Any]] = dict() 

560 

561 # Record if any records needed skipped 

562 resultsTrimmed = False 

563 

564 # Look at each of the metrics in the bundle (name, values) 

565 for metric, measurements in bundle.items(): 

566 # Create a list which will contain the records for each measurement 

567 # associated with metric. 

568 metricRecordList = metricRecords.setdefault(metric, list()) 

569 

570 record: dict[str, Any] = meta.copy() 

571 

572 # loop over each metric measurement within the metric 

573 for measurement in measurements: 

574 # need to extract any tags, package info, etc 

575 note_key = f"{measurement.metric_name.metric}.metric_tags" 

576 record["tags"] = dict(measurement.notes.items()).get(note_key, list()) 

577 

578 # Missing values are replaced by 0 in sasquatch, see RFC-763. 

579 name = "" 

580 value = 0.0 

581 match measurement.json: 

582 case {"metric": name, "value": None}: 

583 pass 

584 case {"metric": name, "value": value}: 

585 if math.isnan(value): 

586 log.error( 

587 "Measurement %s had a value that is a NaN, dispatch will be skipped", 

588 measurement, 

589 ) 

590 resultsTrimmed = True 

591 continue 

592 pass 

593 case {"value": _}: 

594 log.error("Measurement %s does not contain the key 'metric'", measurement) 

595 resultsTrimmed = True 

596 continue 

597 case {"metric": _}: 

598 log.error("Measurement %s does not contain the key 'value'", measurement) 

599 resultsTrimmed = True 

600 continue 

601 record[name] = value 

602 

603 metricRecordList.append({"value": record}) 

604 return metricRecords, resultsTrimmed 

605 

606 def dispatch( 

607 self, 

608 bundle: MetricMeasurementBundle, 

609 run: str, 

610 datasetType: str, 

611 timestamp: datetime.datetime | None = None, 

612 id: UUID | None = None, 

613 datasetIdentifier: str | None = None, 

614 identifierFields: Mapping | None = None, 

615 extraFields: Mapping | None = None, 

616 ) -> None: 

617 """Dispatch a `MetricMeasurementBundle` to Sasquatch. 

618 

619 Parameters 

620 ---------- 

621 bundle : `MetricMeasurementBundle` 

622 The bundle containing metric values to upload. 

623 run : `str` 

624 The run name to associate with these metric values. If this bundle 

625 is also stored in the butler, this should be the butler run 

626 collection the bundle is stored in the butler. This will be used 

627 in generating uniqueness constraints in Sasquatch. 

628 datasetType : `str` 

629 The dataset type name associated with this 

630 `MetricMeasurementBundle`. 

631 timestamp : `datetime.datetime`, optional 

632 The timestamp to be associated with the measurements in the ingress 

633 database. If this value is None, timestamp will be set by the run 

634 time or current time. 

635 id : `UUID`, optional 

636 The UUID of the `MetricMeasurementBundle` within the Butler. If 

637 `None`, a new random UUID will be generated so that each record in 

638 Sasquatch will have a unique value. 

639 datasetIdentifier : `str`, optional 

640 A string which will be used in creating unique identifier tags. If 

641 `None`, a default value will be inserted. 

642 identifierFields: `Mapping`, optional 

643 The keys and values in this mapping will be both added as fields 

644 in the record, and used in creating a unique tag for the uploaded 

645 dataset type. I.e. the timestamp, and the tag will be unique, and 

646 each record will belong to one combination of such. Examples of 

647 entries would be things like visit or tract. 

648 extraFields: `Mapping`, optional 

649 Extra mapping keys and values that will be added as fields to the 

650 dispatched record. 

651 

652 Raises 

653 ------ 

654 SasquatchDispatchPartialFailure 

655 Raised if there were any errors in dispatching a bundle. 

656 """ 

657 if id is None: 

658 id = uuid4() 

659 

660 # Prepare the bundle by transforming it to a list of metric records 

661 metricRecords, recordsTrimmed = self._prepareBundle( 

662 bundle=bundle, 

663 run=run, 

664 datasetType=datasetType, 

665 timestamp=timestamp, 

666 id=id, 

667 datasetIdentifier=datasetIdentifier, 

668 identifierFields=identifierFields, 

669 extraFields=extraFields, 

670 ) 

671 

672 headers = {"content-type": "application/vnd.kafka.avro.v2+json"} 

673 data: dict[str, Any] = dict() 

674 partialUpload = False 

675 uploadFailed = [] 

676 

677 for metric, record in metricRecords.items(): 

678 # create the kafka topic if it does not already exist 

679 if not self._create_topic(metric): 

680 log.error("Topic not created, skipping dispatch of %s", metric) 

681 continue 

682 recordValue = record[0]["value"] 

683 # Generate schemas for each record 

684 data["value_schema"], schemaTrimmed = self._generateAvroSchema(metric, recordValue) 

685 data["records"] = record 

686 

687 if schemaTrimmed: 

688 partialUpload = True 

689 

690 r = requests.post(f"{self.url}/topics/{self.namespace}.{metric}", json=data, headers=headers) 

691 

692 if r.status_code == requests.codes.ok: 

693 log.debug("Succesfully sent data for metric %s", metric) 

694 uploadFailed.append(False) 

695 else: 

696 log.error( 

697 "There was a problem submitting the metric %s: %s, %s", metric, r.status_code, r.json() 

698 ) 

699 uploadFailed.append(True) 

700 partialUpload = True 

701 

702 # There may be no metrics to try to upload, and thus the uploadFailed 

703 # list may be empty, check before issuing failure 

704 if len(uploadFailed) > 0 and all(uploadFailed): 

705 raise SasquatchDispatchFailure("All records were unable to be uploaded.") 

706 

707 if partialUpload or recordsTrimmed: 

708 raise SasquatchDispatchPartialFailure("One or more records may not have been uploaded entirely") 

709 

710 def dispatchRef( 

711 self, 

712 bundle: MetricMeasurementBundle, 

713 ref: DatasetRef, 

714 timestamp: datetime.datetime | None = None, 

715 extraFields: Mapping | None = None, 

716 datasetIdentifier: str | None = None, 

717 ) -> None: 

718 """Dispatch a `MetricMeasurementBundle` to Sasquatch with a known 

719 `DatasetRef`. 

720 

721 Parameters 

722 ---------- 

723 bundle : `MetricMeasurementBundle` 

724 The bundle containing metric values to upload. 

725 ref : `DatasetRef` 

726 The `Butler` dataset ref corresponding to the input 

727 `MetricMeasurementBundle`. 

728 timestamp : `datetime.datetime`, optional 

729 The timestamp to be associated with the measurements in the ingress 

730 database. If this value is None, timestamp will be set by the run 

731 time or current time. 

732 extraFields: `Mapping`, optional 

733 Extra mapping keys and values that will be added as fields to the 

734 dispatched record if not None. 

735 datasetIdentifier : `str`, optional 

736 A string which will be used in creating unique identifier tags. If 

737 None, a default value will be inserted. 

738 

739 Raises 

740 ------ 

741 SasquatchDispatchPartialFailure 

742 Raised if there were any errors in dispatching a bundle. 

743 """ 

744 # Parse the relevant info out of the dataset ref. 

745 serializedRef = ref.to_simple() 

746 id = serializedRef.id 

747 if serializedRef.run is None: 

748 run = "<unknown>" 

749 else: 

750 run = serializedRef.run 

751 dstype = serializedRef.datasetType 

752 datasetType = dstype.name if dstype is not None else "" 

753 dataRefMapping = serializedRef.dataId.dataId if serializedRef.dataId else None 

754 

755 self.dispatch( 

756 bundle, 

757 run=run, 

758 timestamp=timestamp, 

759 datasetType=datasetType, 

760 id=id, 

761 identifierFields=dataRefMapping, 

762 extraFields=extraFields, 

763 datasetIdentifier=datasetIdentifier, 

764 )