Coverage for python/lsst/analysis/tools/interfaces/datastore/_dispatcher.py: 13%
281 statements
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-23 13:13 +0000
« prev ^ index » next coverage.py v7.4.4, created at 2024-03-23 13:13 +0000
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("SasquatchDispatchPartialFailure", "SasquatchDispatchFailure", "SasquatchDispatcher")
26"""Sasquatch datastore"""
27import calendar
28import datetime
29import json
30import logging
31import math
32import re
33from collections.abc import Mapping, MutableMapping, Sequence
34from dataclasses import dataclass
35from typing import TYPE_CHECKING, Any, cast
36from uuid import UUID, uuid4
38import requests
39from lsst.daf.butler import DatasetRef
40from lsst.resources import ResourcePath
41from lsst.utils.packages import getEnvironmentPackages
43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true
44 from .. import MetricMeasurementBundle
47log = logging.getLogger(__name__)
49# Constants assocated with SasquatchDispatcher
50PARTITIONS = 1
51REPLICATION_FACTOR = 3
53IDENTIFIER_KEYS = [
54 "detector",
55 "patch",
56 "skymap",
57 "visit",
58 "tract",
59 "physical_filter",
60 "instrument",
61 "band",
62 "exposure",
63 "group",
64 "day_obs",
65]
68class SasquatchDispatchPartialFailure(RuntimeError):
69 """This indicates that a Sasquatch dispatch was partially successful."""
71 pass
74class SasquatchDispatchFailure(RuntimeError):
75 """This indicates that dispatching a
76 `~lsst.analysis.tool.interface.MetricMeasurementBundle` failed.
77 """
79 pass
82def _tag2VersionTime(productStr: str) -> tuple[str, float]:
83 """Determine versions and dates from the string returned from
84 getEnvironmentPackages.
86 The `~lsst.utils.packages.genEnvironmentPackages` function returns the
87 setup version associated with a product, along with a list of tags that
88 have been added to it.
90 This method splits up that return string, and determines the earliest date
91 associated with the setup package version.
93 Parameters
94 ----------
95 productStr : `str`
96 The product string returned from a lookup on the result of a call to
97 `~lsst.utils.packages.getEnvironmentPackages`.
99 Returns
100 -------
101 result : `tuple` of `str`, `datetime.datetime`
102 The first `str` is the version of the package, and the second is the
103 datetime object associated with that released version.
105 Raises
106 ------
107 ValueError
108 Raised if there are no tags which correspond to dates.
109 """
110 times: list[datetime.datetime] = []
111 version = productStr.split()[0]
112 tags: str = re.findall("[(](.*)[)]", productStr)[0]
113 for tag in tags.split():
114 numDots = tag.count(".")
115 numUnder = tag.count("_")
116 separator = "_"
117 if numDots > numUnder:
118 separator = "."
119 match tag.split(separator):
120 # Daily tag branch.
121 case ("d", year, month, day):
122 dt = datetime.datetime(year=int(year), month=int(month), day=int(day))
123 # Weekly tag branch.
124 case ("w", year, week):
125 iyear = int(year)
126 iweek = int(week)
127 # Use 4 as the day because releases are available starting
128 # on Thursday
129 dayOfWeek = 4
131 # Find the first week to contain a thursday in it
132 cal = calendar.Calendar()
133 cal.setfirstweekday(6)
134 i = 0
135 for i, iterWeek in enumerate(cal.monthdatescalendar(iyear, 1)):
136 if iterWeek[dayOfWeek].month == 1:
137 break
138 # Handle fromisocalendar not being able to handle week 53
139 # in the case were the date was going to subtract 7 days anyway
140 if i and iweek == 53:
141 i = 0
142 iweek = 52
143 delta = datetime.timedelta(days=7 * i)
145 # Correct for a weekly being issued in the last week of the
146 # previous year, as Thursdays don't always line up evenly in
147 # a week / year split.
148 dt = datetime.datetime.fromisocalendar(iyear, iweek, dayOfWeek) - delta
149 # Skip tags that can't be understood.
150 case _:
151 continue
152 times.append(dt)
153 if len(times) == 0:
154 raise ValueError("Could not find any tags corresponding to dates")
155 minTime = min(times)
156 minTime.replace(tzinfo=datetime.timezone.utc)
157 return version, minTime.timestamp()
160@dataclass
161class SasquatchDispatcher:
162 """This class mediates the transfer of MetricMeasurementBundles to a
163 Sasquatch http kafka proxy server.
164 """
166 url: str
167 """Url of the Sasquatch proxy server"""
169 token: str
170 """Authentication token used in communicating with the proxy server"""
172 namespace: str = "lsst.dm"
173 """The namespace in Sasquatch in which to write the uploaded metrics"""
175 def __post_init__(self) -> None:
176 match ResourcePath(self.url).scheme:
177 case "http" | "https":
178 pass
179 case _:
180 raise ValueError("Proxy server must be locatable with either http or https")
182 self._cluster_id: str | None = None
184 @property
185 def clusterId(self) -> str:
186 """ClusterId of the Kafka proxy
188 Notes
189 -----
190 The cluster Id will be fetched with a network call if it is not
191 already cached.
192 """
193 if self._cluster_id is None:
194 self._populateClusterId()
195 return cast(str, self._cluster_id)
197 def _populateClusterId(self) -> None:
198 """Get Sasquatch kafka cluster ID."""
200 headers = {"content-type": "application/json"}
201 r = requests.get(f"{self.url}/v3/clusters", headers=headers)
203 if r.status_code == requests.codes.ok:
204 cluster_id = r.json()["data"][0]["cluster_id"]
206 self._cluster_id = str(cluster_id)
207 else:
208 log.error("Could not retrieve the cluster id for the specified url")
209 raise SasquatchDispatchFailure("Could not retrieve the cluster id for the specified url")
211 def _create_topic(self, topic_name: str) -> bool:
212 """Create a kafka topic in Sasquatch.
214 Parameters
215 ----------
216 topic_name : `str`
217 The name of the kafka topic to create
219 Returns
220 -------
221 status : `bool`
222 If this does not encounter an error it will return a True success
223 code, else it will return a False code.
225 """
227 headers = {"content-type": "application/json"}
229 topic_config = {
230 "topic_name": f"{self.namespace}.{topic_name}",
231 "partitions_count": PARTITIONS,
232 "replication_factor": REPLICATION_FACTOR,
233 }
235 r = requests.post(
236 f"{self.url}/v3/clusters/{self.clusterId}/topics", json=topic_config, headers=headers
237 )
239 if r.status_code == requests.codes.created:
240 log.debug("Created topic %s.%s", self.namespace, topic_name)
241 return True
242 elif r.status_code == requests.codes.bad_request:
243 log.debug("Topic %s.%s already exists.", self.namespace, topic_name)
244 return True
245 else:
246 log.error("Uknown error occured creating kafka topic %s %s", r.status_code, r.json())
247 return False
249 def _generateAvroSchema(self, metric: str, record: MutableMapping[str, Any]) -> tuple[str, bool]:
250 """Infer the Avro schema from the record payload.
252 Parameters
253 ----------
254 metric : `str`
255 The name of the metric
256 record : `MutableMapping`
257 The prepared record for which a schema is to be generated
259 Returns
260 -------
261 resultSchema : `str`
262 A json encoded string of the resulting avro schema
263 errorCode : bool
264 A boolean indicating if any record fields had to be trimmed because
265 a suitable schema could not be generated. True if records were
266 removed, False otherwise.
267 """
268 schema: dict[str, Any] = {"type": "record", "namespace": self.namespace, "name": metric}
270 # Record if any records needed to be trimmed
271 resultsTrimmed = False
273 fields = list()
274 # If avro schemas cant be generated for values, they should be removed
275 # from the records.
276 keysToRemove: list[str] = []
277 for key in record:
278 value = record[key]
279 avroType: Mapping[str, Any]
280 if "timestamp" in key:
281 avroType = {"type": "double"}
282 else:
283 avroType = self._python2Avro(value)
284 if len(avroType) == 0:
285 continue
286 if avroType.get("error_in_conversion"):
287 keysToRemove.append(key)
288 resultsTrimmed = True
289 continue
290 fields.append({"name": key, **avroType})
292 # remove any key that failed to have schema generated
293 for key in keysToRemove:
294 record.pop(key)
296 schema["fields"] = fields
298 return json.dumps(schema), resultsTrimmed
300 def _python2Avro(self, value: Any) -> Mapping:
301 """Map python type to avro schema
303 Parameters
304 ----------
305 value : `Any`
306 Any python parameter.
308 Returns
309 -------
310 result : `Mapping`
311 Return a mapping that represents an entry in an avro schema.
312 """
313 match value:
314 case float() | None:
315 return {"type": "float", "default": 0.0}
316 case str():
317 return {"type": "string", "default": ""}
318 case int():
319 return {"type": "int", "default": 0}
320 case Sequence():
321 tmp = {self._python2Avro(item)["type"] for item in value}
322 if len(tmp) == 0:
323 return {}
324 if len(tmp) > 1:
325 log.error(
326 "Sequence contains mixed types: %s, must be homogeneous for avro conversion "
327 "skipping record",
328 tmp,
329 )
330 return {"error_in_conversion": True}
331 return {"type": "array", "items": tmp.pop()}
332 case _:
333 log.error("Unsupported type %s, skipping record", type(value))
334 return {}
336 def _handleReferencePackage(self, meta: MutableMapping, bundle: MetricMeasurementBundle) -> None:
337 """Check to see if there is a reference package.
339 if there is a reference package, determine the datetime associated with
340 this reference package. Save the package, the version, and the date to
341 the common metric fields.
343 Parameters
344 ----------
345 meta : `MutableMapping`
346 A mapping which corresponds to fields which should be encoded in
347 all records.
348 bundle : `MetricMeasurementBundle`
349 The bundled metrics
350 """
351 package_version, package_timestamp = "", 0.0
352 if ref_package := getattr(bundle, "reference_package", ""):
353 ref_package = bundle.reference_package
354 packages = getEnvironmentPackages(True)
355 if package_info := packages.get(ref_package):
356 try:
357 package_version, package_timestamp = _tag2VersionTime(package_info)
358 except ValueError:
359 # Could not extract package timestamp leaving empty
360 pass
361 # explicit handle if None was set in the bundle for the package
362 meta["reference_package"] = ref_package or ""
363 meta["reference_package_version"] = package_version
364 meta["reference_package_timestamp"] = package_timestamp
366 def _handleTimes(self, meta: MutableMapping, bundle: MetricMeasurementBundle, run: str) -> None:
367 """Add times to the meta fields mapping.
369 Add all appropriate timestamp fields to the meta field mapping. These
370 will be added to all records.
372 This method will also look at the bundle to see if it defines a
373 preferred time. It so it sets that time as the main time stamp to be
374 used for this record.
376 Parameters
377 ----------
378 meta : `MutableMapping`
379 A mapping which corresponds to fields which should be encoded in
380 all records.
381 bundle : `MetricMeasurementBundle`
382 The bundled metrics
383 run : `str`
384 The `~lsst.daf.butler.Butler` collection where the
385 `MetricMeasurementBundle` is stored.
386 """
387 # Determine the timestamp associated with the run, if someone abused
388 # the run collection, use the current timestamp
389 if re.match(r"\d{8}T\d{6}Z", stamp := run.split("/")[-1]):
390 run_timestamp = datetime.datetime.strptime(stamp, r"%Y%m%dT%H%M%S%z")
391 else:
392 run_timestamp = datetime.datetime.now()
393 meta["run_timestamp"] = run_timestamp.timestamp()
395 # If the bundle supports supplying timestamps, dispatch on the type
396 # specified.
397 if hasattr(bundle, "timestamp_version") and bundle.timestamp_version:
398 match bundle.timestamp_version:
399 case "reference_package_timestamp":
400 if not meta["reference_package_timestamp"]:
401 log.error("Reference package timestamp is empty, using run_timestamp")
402 meta["timestamp"] = meta["run_timestamp"]
403 else:
404 meta["timestamp"] = meta["reference_package_timestamp"]
405 case "run_timestamp":
406 meta["timestamp"] = meta["run_timestamp"]
407 case "current_timestamp":
408 timeStamp = datetime.datetime.now()
409 meta["timestamp"] = timeStamp.timestamp()
410 case "dataset_timestamp":
411 log.error("dataset timestamps are not yet supported, run_timestamp will be used")
412 meta["timestamp"] = meta["run_timestamp"]
413 case str(value) if "explicit_timestamp" in value:
414 try:
415 _, splitTime = value.split(":")
416 except ValueError as excpt:
417 raise ValueError(
418 "Explicit timestamp must be given in the format 'explicit_timestamp:datetime', "
419 "where datetime is given in the form '%Y%m%dT%H%M%S%z"
420 ) from excpt
421 meta["timestamp"] = datetime.datetime.strptime(splitTime, r"%Y%m%dT%H%M%S%z").timestamp()
422 case _:
423 log.error(
424 "Timestamp version %s is not supported, run_timestamp will be used",
425 bundle.timestamp_version,
426 )
427 meta["timestamp"] = meta["run_timestamp"]
428 # Default to using the run_timestamp.
429 else:
430 meta["timestamp"] = meta["run_timestamp"]
432 def _handleIdentifier(
433 self,
434 meta: MutableMapping,
435 identifierFields: Mapping[str, Any] | None,
436 datasetIdentifier: str | None,
437 bundle: MetricMeasurementBundle,
438 ) -> None:
439 """Add an identifier to the meta record mapping.
441 If the bundle declares a dataset identifier to use add that to the
442 record, otherwise use 'Generic' as the identifier. If the
443 datasetIdentifier parameter is specified, that is used instead of
444 anything specified by the bundle.
446 This will also add any identifier fields supplied to the meta record
447 mapping.
449 Together these values (in addition to the timestamp and topic) should
450 uniquely identify an upload to the Sasquatch system.
452 Parameters
453 ----------
454 meta : `MutableMapping`
455 A mapping which corresponds to fields which should be encoded in
456 all records.
457 identifierFields: `Mapping` or `None`
458 The keys and values in this mapping will be both added as fields
459 in the record, and used in creating a unique tag for the uploaded
460 dataset type. I.e. the timestamp, and the tag will be unique, and
461 each record will belong to one combination of such.
462 datasetIdentifier : `str` or `None`
463 A string which will be used in creating unique identifier tags.
464 bundle : `MetricMeasurementBundle`
465 The bundle containing metric values to upload.
466 """
467 identifier: str
468 if datasetIdentifier is not None:
469 identifier = datasetIdentifier
470 elif hasattr(bundle, "dataset_identifier") and bundle.dataset_identifier is not None:
471 identifier = bundle.dataset_identifier
472 else:
473 identifier = "Generic"
475 meta["dataset_tag"] = identifier
477 if identifierFields is None:
478 identifierFields = {}
479 for key in IDENTIFIER_KEYS:
480 value = identifierFields.get(key, "")
481 meta[key] = f"{value}"
483 def _prepareBundle(
484 self,
485 bundle: MetricMeasurementBundle,
486 run: str,
487 datasetType: str,
488 timestamp: datetime.datetime | None = None,
489 id: UUID | None = None,
490 identifierFields: Mapping | None = None,
491 datasetIdentifier: str | None = None,
492 extraFields: Mapping | None = None,
493 ) -> tuple[Mapping[str, list[Any]], bool]:
494 """Encode all of the inputs into a format that can be sent to the
495 kafka proxy server.
497 Parameters
498 ----------
499 bundle : `MetricMeasurementBundle`
500 The bundle containing metric values to upload.
501 run : `str`
502 The run name to associate with these metric values. If this bundle
503 is also stored in the butler, this should be the butler run
504 collection the bundle is stored in the butler.
505 datasetType : `str`
506 The dataset type name associated with this
507 `MetricMeasurementBundle`
508 timestamp : `datetime.datetime`, optional
509 The timestamp to be associated with the measurements in the ingress
510 database. If this value is None, timestamp will be set by the run
511 time or current time.
512 id : `UUID`, optional
513 The UUID of the `MetricMeasurementBundle` within the butler. If
514 `None`, a new random UUID will be generated so that each record in
515 Sasquatch will have a unique value.
516 identifierFields: `Mapping`, optional
517 The keys and values in this mapping will be both added as fields
518 in the record, and used in creating a unique tag for the uploaded
519 dataset type. I.e. the timestamp, and the tag will be unique, and
520 each record will belong to one combination of such.
521 datasetIdentifier : `str`, optional
522 A string which will be used in creating unique identifier tags.
523 extraFields: `Mapping`, optional
524 Extra mapping keys and values that will be added as fields to the
525 dispatched record.
527 Returns
528 -------
529 result : `Mapping` of `str` to `list`
530 A mapping of metric name of list of metric measurement records.
531 status : `bool`
532 A status boolean indicating if some records had to be skipped due
533 to a problem parsing the bundle.
534 """
535 if id is None:
536 id = uuid4()
537 sid = str(id)
538 meta: dict[str, Any] = dict()
540 # Add other associated common fields
541 meta["id"] = sid
542 meta["run"] = run
543 meta["dataset_type"] = datasetType
545 # Check to see if the bundle declares a reference package
546 self._handleReferencePackage(meta, bundle)
548 # Handle the various timestamps that could be associated with a record
549 self._handleTimes(meta, bundle, run)
551 # Always use the supplied timestamp if one was passed to use.
552 if timestamp is not None:
553 meta["timestamp"] = timestamp.timestamp()
555 self._handleIdentifier(meta, identifierFields, datasetIdentifier, bundle)
557 # Add in any other fields that were supplied to the function call.
558 if extraFields is not None:
559 meta.update(extraFields)
561 metricRecords: dict[str, list[Any]] = dict()
563 # Record if any records needed skipped
564 resultsTrimmed = False
566 # Look at each of the metrics in the bundle (name, values)
567 for metric, measurements in bundle.items():
568 # Create a list which will contain the records for each measurement
569 # associated with metric.
570 metricRecordList = metricRecords.setdefault(metric, list())
572 record: dict[str, Any] = meta.copy()
574 # loop over each metric measurement within the metric
575 for measurement in measurements:
576 # need to extract any tags, package info, etc
577 note_key = f"{measurement.metric_name.metric}.metric_tags"
578 record["tags"] = dict(measurement.notes.items()).get(note_key, list())
580 # Missing values are replaced by 0 in sasquatch, see RFC-763.
581 name = ""
582 value = 0.0
583 match measurement.json:
584 case {"metric": name, "value": None}:
585 pass
586 case {"metric": name, "value": value}:
587 if math.isnan(value):
588 log.error(
589 "Measurement %s had a value that is a NaN, dispatch will be skipped",
590 measurement,
591 )
592 resultsTrimmed = True
593 continue
594 pass
595 case {"value": _}:
596 log.error("Measurement %s does not contain the key 'metric'", measurement)
597 resultsTrimmed = True
598 continue
599 case {"metric": _}:
600 log.error("Measurement %s does not contain the key 'value'", measurement)
601 resultsTrimmed = True
602 continue
603 record[name] = value
605 metricRecordList.append({"value": record})
606 return metricRecords, resultsTrimmed
608 def dispatch(
609 self,
610 bundle: MetricMeasurementBundle,
611 run: str,
612 datasetType: str,
613 timestamp: datetime.datetime | None = None,
614 id: UUID | None = None,
615 datasetIdentifier: str | None = None,
616 identifierFields: Mapping | None = None,
617 extraFields: Mapping | None = None,
618 ) -> None:
619 """Dispatch a `MetricMeasurementBundle` to Sasquatch.
621 Parameters
622 ----------
623 bundle : `MetricMeasurementBundle`
624 The bundle containing metric values to upload.
625 run : `str`
626 The run name to associate with these metric values. If this bundle
627 is also stored in the butler, this should be the butler run
628 collection the bundle is stored in the butler. This will be used
629 in generating uniqueness constraints in Sasquatch.
630 datasetType : `str`
631 The dataset type name associated with this
632 `MetricMeasurementBundle`.
633 timestamp : `datetime.datetime`, optional
634 The timestamp to be associated with the measurements in the ingress
635 database. If this value is None, timestamp will be set by the run
636 time or current time.
637 id : `UUID`, optional
638 The UUID of the `MetricMeasurementBundle` within the Butler. If
639 `None`, a new random UUID will be generated so that each record in
640 Sasquatch will have a unique value.
641 datasetIdentifier : `str`, optional
642 A string which will be used in creating unique identifier tags. If
643 `None`, a default value will be inserted.
644 identifierFields: `Mapping`, optional
645 The keys and values in this mapping will be both added as fields
646 in the record, and used in creating a unique tag for the uploaded
647 dataset type. I.e. the timestamp, and the tag will be unique, and
648 each record will belong to one combination of such. Examples of
649 entries would be things like visit or tract.
650 extraFields: `Mapping`, optional
651 Extra mapping keys and values that will be added as fields to the
652 dispatched record.
654 Raises
655 ------
656 SasquatchDispatchPartialFailure
657 Raised if there were any errors in dispatching a bundle.
658 """
659 if id is None:
660 id = uuid4()
662 # Prepare the bundle by transforming it to a list of metric records
663 metricRecords, recordsTrimmed = self._prepareBundle(
664 bundle=bundle,
665 run=run,
666 datasetType=datasetType,
667 timestamp=timestamp,
668 id=id,
669 datasetIdentifier=datasetIdentifier,
670 identifierFields=identifierFields,
671 extraFields=extraFields,
672 )
674 headers = {"content-type": "application/vnd.kafka.avro.v2+json"}
675 data: dict[str, Any] = dict()
676 partialUpload = False
677 uploadFailed = []
679 for metric, record in metricRecords.items():
680 # create the kafka topic if it does not already exist
681 if not self._create_topic(metric):
682 log.error("Topic not created, skipping dispatch of %s", metric)
683 continue
684 recordValue = record[0]["value"]
685 # Generate schemas for each record
686 data["value_schema"], schemaTrimmed = self._generateAvroSchema(metric, recordValue)
687 data["records"] = record
689 if schemaTrimmed:
690 partialUpload = True
692 r = requests.post(f"{self.url}/topics/{self.namespace}.{metric}", json=data, headers=headers)
694 if r.status_code == requests.codes.ok:
695 log.debug("Succesfully sent data for metric %s", metric)
696 uploadFailed.append(False)
697 else:
698 log.error(
699 "There was a problem submitting the metric %s: %s, %s", metric, r.status_code, r.json()
700 )
701 uploadFailed.append(True)
702 partialUpload = True
704 # There may be no metrics to try to upload, and thus the uploadFailed
705 # list may be empty, check before issuing failure
706 if len(uploadFailed) > 0 and all(uploadFailed):
707 raise SasquatchDispatchFailure("All records were unable to be uploaded.")
709 if partialUpload or recordsTrimmed:
710 raise SasquatchDispatchPartialFailure("One or more records may not have been uploaded entirely")
712 def dispatchRef(
713 self,
714 bundle: MetricMeasurementBundle,
715 ref: DatasetRef,
716 timestamp: datetime.datetime | None = None,
717 extraFields: Mapping | None = None,
718 datasetIdentifier: str | None = None,
719 ) -> None:
720 """Dispatch a `MetricMeasurementBundle` to Sasquatch with a known
721 `DatasetRef`.
723 Parameters
724 ----------
725 bundle : `MetricMeasurementBundle`
726 The bundle containing metric values to upload.
727 ref : `DatasetRef`
728 The `Butler` dataset ref corresponding to the input
729 `MetricMeasurementBundle`.
730 timestamp : `datetime.datetime`, optional
731 The timestamp to be associated with the measurements in the ingress
732 database. If this value is None, timestamp will be set by the run
733 time or current time.
734 extraFields: `Mapping`, optional
735 Extra mapping keys and values that will be added as fields to the
736 dispatched record if not None.
737 datasetIdentifier : `str`, optional
738 A string which will be used in creating unique identifier tags. If
739 None, a default value will be inserted.
741 Raises
742 ------
743 SasquatchDispatchPartialFailure
744 Raised if there were any errors in dispatching a bundle.
745 """
746 # Parse the relevant info out of the dataset ref.
747 serializedRef = ref.to_simple()
748 id = serializedRef.id
749 if serializedRef.run is None:
750 run = "<unknown>"
751 else:
752 run = serializedRef.run
753 dstype = serializedRef.datasetType
754 datasetType = dstype.name if dstype is not None else ""
755 dataRefMapping = serializedRef.dataId.dataId if serializedRef.dataId else None
757 self.dispatch(
758 bundle,
759 run=run,
760 timestamp=timestamp,
761 datasetType=datasetType,
762 id=id,
763 identifierFields=dataRefMapping,
764 extraFields=extraFields,
765 datasetIdentifier=datasetIdentifier,
766 )