Coverage for python/lsst/analysis/tools/interfaces/datastore/_dispatcher.py: 13%
275 statements
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 18:03 +0000
« prev ^ index » next coverage.py v7.3.2, created at 2023-10-25 18:03 +0000
1# This file is part of analysis_tools.
2#
3# Developed for the LSST Data Management System.
4# This product includes software developed by the LSST Project
5# (http://www.lsst.org).
6# See the COPYRIGHT file at the top-level directory of this distribution
7# for details of code ownership.
8#
9# This program is free software: you can redistribute it and/or modify
10# it under the terms of the GNU General Public License as published by
11# the Free Software Foundation, either version 3 of the License, or
12# (at your option) any later version.
13#
14# This program is distributed in the hope that it will be useful,
15# but WITHOUT ANY WARRANTY; without even the implied warranty of
16# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
17# GNU General Public License for more details.
18#
19# You should have received a copy of the GNU General Public License
20# along with this program. If not, see <http://www.gnu.org/licenses/>.
22from __future__ import annotations
24__all__ = ("SasquatchDispatchPartialFailure", "SasquatchDispatchFailure", "SasquatchDispatcher")
26"""Sasquatch datastore"""
27import calendar
28import datetime
29import json
30import logging
31import math
32import re
33from collections.abc import Mapping, MutableMapping, Sequence
34from dataclasses import dataclass
35from typing import TYPE_CHECKING, Any, cast
36from uuid import UUID, uuid4
38import requests
39from lsst.daf.butler import DatasetRef
40from lsst.resources import ResourcePath
41from lsst.utils.packages import getEnvironmentPackages
43if TYPE_CHECKING: 43 ↛ 44line 43 didn't jump to line 44, because the condition on line 43 was never true
44 from .. import MetricMeasurementBundle
47log = logging.getLogger(__name__)
49# Constants assocated with SasquatchDispatcher
50PARTITIONS = 1
51REPLICATION_FACTOR = 3
53IDENTIFIER_KEYS = [
54 "detector",
55 "patch",
56 "skymap",
57 "visit",
58 "tract",
59 "physical_filter",
60 "instrument",
61 "band",
62 "exposure",
63]
66class SasquatchDispatchPartialFailure(RuntimeError):
67 """This indicates that a Sasquatch dispatch was partially successful."""
69 pass
72class SasquatchDispatchFailure(RuntimeError):
73 """This indicates that dispatching a
74 `~lsst.analysis.tool.interface.MetricMeasurementBundle` failed.
75 """
77 pass
80def _tag2VersionTime(productStr: str) -> tuple[str, float]:
81 """Determine versions and dates from the string returned from
82 getEnvironmentPackages.
84 The `~lsst.utils.packages.genEnvironmentPackages` function returns the
85 setup version associated with a product, along with a list of tags that
86 have been added to it.
88 This method splits up that return string, and determines the earliest date
89 associated with the setup package version.
91 Parameters
92 ----------
93 productStr : `str`
94 The product string returned from a lookup on the result of a call to
95 `~lsst.utils.packages.getEnvironmentPackages`.
97 Returns
98 -------
99 result : `tuple` of `str`, `datetime.datetime`
100 The first `str` is the version of the package, and the second is the
101 datetime object associated with that released version.
103 Raises
104 ------
105 ValueError
106 Raised if there are no tags which correspond to dates.
107 """
108 times: list[datetime.datetime] = []
109 version = productStr.split()[0]
110 tags: str = re.findall("[(](.*)[)]", productStr)[0]
111 for tag in tags.split():
112 numDots = tag.count(".")
113 numUnder = tag.count("_")
114 separator = "_"
115 if numDots > numUnder:
116 separator = "."
117 match tag.split(separator):
118 # Daily tag branch.
119 case ("d", year, month, day):
120 dt = datetime.datetime(year=int(year), month=int(month), day=int(day))
121 # Weekly tag branch.
122 case ("w", year, week):
123 iyear = int(year)
124 iweek = int(week)
125 # Use 4 as the day because releases are available starting
126 # on Thursday
127 dayOfWeek = 4
129 # Find the first week to contain a thursday in it
130 cal = calendar.Calendar()
131 cal.setfirstweekday(6)
132 i = 0
133 for i, iterWeek in enumerate(cal.monthdatescalendar(iyear, 1)):
134 if iterWeek[dayOfWeek].month == 1:
135 break
136 # Handle fromisocalendar not being able to handle week 53
137 # in the case were the date was going to subtract 7 days anyway
138 if i and iweek == 53:
139 i = 0
140 iweek = 52
141 delta = datetime.timedelta(days=7 * i)
143 # Correct for a weekly being issued in the last week of the
144 # previous year, as Thursdays don't always line up evenly in
145 # a week / year split.
146 dt = datetime.datetime.fromisocalendar(iyear, iweek, dayOfWeek) - delta
147 # Skip tags that can't be understood.
148 case _:
149 continue
150 times.append(dt)
151 if len(times) == 0:
152 raise ValueError("Could not find any tags corresponding to dates")
153 minTime = min(times)
154 minTime.replace(tzinfo=datetime.timezone.utc)
155 return version, minTime.timestamp()
158@dataclass
159class SasquatchDispatcher:
160 """This class mediates the transfer of MetricMeasurementBundles to a
161 Sasquatch http kafka proxy server.
162 """
164 url: str
165 """Url of the Sasquatch proxy server"""
167 token: str
168 """Authentication token used in communicating with the proxy server"""
170 namespace: str = "lsst.dm"
171 """The namespace in Sasquatch in which to write the uploaded metrics"""
173 def __post_init__(self) -> None:
174 match ResourcePath(self.url).scheme:
175 case "http" | "https":
176 pass
177 case _:
178 raise ValueError("Proxy server must be locatable with either http or https")
180 self._cluster_id: str | None = None
182 @property
183 def clusterId(self) -> str:
184 """ClusterId of the Kafka proxy
186 Notes
187 -----
188 The cluster Id will be fetched with a network call if it is not
189 already cached.
190 """
191 if self._cluster_id is None:
192 self._populateClusterId()
193 return cast(str, self._cluster_id)
195 def _populateClusterId(self) -> None:
196 """Get Sasquatch kafka cluster ID."""
198 headers = {"content-type": "application/json"}
199 r = requests.get(f"{self.url}/v3/clusters", headers=headers)
201 if r.status_code == requests.codes.ok:
202 cluster_id = r.json()["data"][0]["cluster_id"]
204 self._cluster_id = str(cluster_id)
205 else:
206 log.error("Could not retrieve the cluster id for the specified url")
207 raise SasquatchDispatchFailure("Could not retrieve the cluster id for the specified url")
209 def _create_topic(self, topic_name: str) -> bool:
210 """Create a kafka topic in Sasquatch.
212 Parameters
213 ----------
214 topic_name : `str`
215 The name of the kafka topic to create
217 Returns
218 -------
219 status : `bool`
220 If this does not encounter an error it will return a True success
221 code, else it will return a False code.
223 """
225 headers = {"content-type": "application/json"}
227 topic_config = {
228 "topic_name": f"{self.namespace}.{topic_name}",
229 "partitions_count": PARTITIONS,
230 "replication_factor": REPLICATION_FACTOR,
231 }
233 r = requests.post(
234 f"{self.url}/v3/clusters/{self.clusterId}/topics", json=topic_config, headers=headers
235 )
237 if r.status_code == requests.codes.created:
238 log.debug("Created topic %s.%s", self.namespace, topic_name)
239 return True
240 elif r.status_code == requests.codes.bad_request:
241 log.debug("Topic %s.%s already exists.", self.namespace, topic_name)
242 return True
243 else:
244 log.error("Uknown error occured creating kafka topic %s %s", r.status_code, r.json())
245 return False
247 def _generateAvroSchema(self, metric: str, record: MutableMapping[str, Any]) -> tuple[str, bool]:
248 """Infer the Avro schema from the record payload.
250 Parameters
251 ----------
252 metric : `str`
253 The name of the metric
254 record : `MutableMapping`
255 The prepared record for which a schema is to be generated
257 Returns
258 -------
259 resultSchema : `str`
260 A json encoded string of the resulting avro schema
261 errorCode : bool
262 A boolean indicating if any record fields had to be trimmed because
263 a suitable schema could not be generated. True if records were
264 removed, False otherwise.
265 """
266 schema: dict[str, Any] = {"type": "record", "namespace": self.namespace, "name": metric}
268 # Record if any records needed to be trimmed
269 resultsTrimmed = False
271 fields = list()
272 # If avro schemas cant be generated for values, they should be removed
273 # from the records.
274 keysToRemove: list[str] = []
275 for key in record:
276 value = record[key]
277 avroType: Mapping[str, Any]
278 if "timestamp" in key:
279 avroType = {"type": "double"}
280 else:
281 avroType = self._python2Avro(value)
282 if len(avroType) == 0:
283 continue
284 if avroType.get("error_in_conversion"):
285 keysToRemove.append(key)
286 resultsTrimmed = True
287 continue
288 fields.append({"name": key, **avroType})
290 # remove any key that failed to have schema generated
291 for key in keysToRemove:
292 record.pop(key)
294 schema["fields"] = fields
296 return json.dumps(schema), resultsTrimmed
298 def _python2Avro(self, value: Any) -> Mapping:
299 """Map python type to avro schema
301 Parameters
302 ----------
303 value : `Any`
304 Any python parameter.
306 Returns
307 -------
308 result : `Mapping`
309 Return a mapping that represents an entry in an avro schema.
310 """
311 match value:
312 case float() | None:
313 return {"type": "float", "default": 0.0}
314 case str():
315 return {"type": "string", "default": ""}
316 case int():
317 return {"type": "int", "default": 0}
318 case Sequence():
319 tmp = {self._python2Avro(item)["type"] for item in value}
320 if len(tmp) == 0:
321 return {}
322 if len(tmp) > 1:
323 log.error(
324 "Sequence contains mixed types: %s, must be homogeneous for avro conversion "
325 "skipping record",
326 tmp,
327 )
328 return {"error_in_conversion": True}
329 return {"type": "array", "items": tmp.pop()}
330 case _:
331 log.error("Unsupported type %s, skipping record", type(value))
332 return {}
334 def _handleReferencePackage(self, meta: MutableMapping, bundle: MetricMeasurementBundle) -> None:
335 """Check to see if there is a reference package.
337 if there is a reference package, determine the datetime associated with
338 this reference package. Save the package, the version, and the date to
339 the common metric fields.
341 Parameters
342 ----------
343 meta : `MutableMapping`
344 A mapping which corresponds to fields which should be encoded in
345 all records.
346 bundle : `MetricMeasurementBundle`
347 The bundled metrics
348 """
349 package_version, package_timestamp = "", 0.0
350 if ref_package := getattr(bundle, "reference_package", ""):
351 ref_package = bundle.reference_package
352 packages = getEnvironmentPackages(True)
353 if package_info := packages.get(ref_package):
354 try:
355 package_version, package_timestamp = _tag2VersionTime(package_info)
356 except ValueError:
357 # Could not extract package timestamp leaving empty
358 pass
359 # explicit handle if None was set in the bundle for the package
360 meta["reference_package"] = ref_package or ""
361 meta["reference_package_version"] = package_version
362 meta["reference_package_timestamp"] = package_timestamp
364 def _handleTimes(self, meta: MutableMapping, bundle: MetricMeasurementBundle, run: str) -> None:
365 """Add times to the meta fields mapping.
367 Add all appropriate timestamp fields to the meta field mapping. These
368 will be added to all records.
370 This method will also look at the bundle to see if it defines a
371 preferred time. It so it sets that time as the main time stamp to be
372 used for this record.
374 Parameters
375 ----------
376 meta : `MutableMapping`
377 A mapping which corresponds to fields which should be encoded in
378 all records.
379 bundle : `MetricMeasurementBundle`
380 The bundled metrics
381 run : `str`
382 The `~lsst.daf.butler.Butler` collection where the
383 `MetricMeasurementBundle` is stored.
384 """
385 # Determine the timestamp associated with the run, if someone abused
386 # the run collection, use the current timestamp
387 if re.match(r"\d{8}T\d{6}Z", stamp := run.split("/")[-1]):
388 run_timestamp = datetime.datetime.strptime(stamp, r"%Y%m%dT%H%M%S%z")
389 else:
390 run_timestamp = datetime.datetime.now()
391 meta["run_timestamp"] = run_timestamp.timestamp()
393 # If the bundle supports supplying timestamps, dispatch on the type
394 # specified.
395 if hasattr(bundle, "timestamp_version") and bundle.timestamp_version:
396 match bundle.timestamp_version:
397 case "reference_package_timestamp":
398 if not meta["reference_package_timestamp"]:
399 log.error("Reference package timestamp is empty, using run_timestamp")
400 meta["timestamp"] = meta["run_timestamp"]
401 else:
402 meta["timestamp"] = meta["reference_package_timestamp"]
403 case "run_timestamp":
404 meta["timestamp"] = meta["run_timestamp"]
405 case "current_timestamp":
406 timeStamp = datetime.datetime.now()
407 meta["timestamp"] = timeStamp.timestamp()
408 case "dataset_timestamp":
409 log.error("dataset timestamps are not yet supported, run_timestamp will be used")
410 meta["timestamp"] = meta["run_timestamp"]
411 case _:
412 log.error(
413 "Timestamp version %s is not supported, run_timestamp will be used",
414 bundle.timestamp_version,
415 )
416 meta["timestamp"] = meta["run_timestamp"]
417 # Default to using the run_timestamp.
418 else:
419 meta["timestamp"] = meta["run_timestamp"]
421 def _handleIdentifier(
422 self,
423 meta: MutableMapping,
424 identifierFields: Mapping[str, Any] | None,
425 datasetIdentifier: str | None,
426 bundle: MetricMeasurementBundle,
427 ) -> None:
428 """Add an identifier to the meta record mapping.
430 If the bundle declares a dataset identifier to use add that to the
431 record, otherwise use 'Generic' as the identifier. If the
432 datasetIdentifier parameter is specified, that is used instead of
433 anything specified by the bundle.
435 This will also add any identifier fields supplied to the meta record
436 mapping.
438 Together these values (in addition to the timestamp and topic) should
439 uniquely identify an upload to the Sasquatch system.
441 Parameters
442 ----------
443 meta : `MutableMapping`
444 A mapping which corresponds to fields which should be encoded in
445 all records.
446 identifierFields: `Mapping` or `None`
447 The keys and values in this mapping will be both added as fields
448 in the record, and used in creating a unique tag for the uploaded
449 dataset type. I.e. the timestamp, and the tag will be unique, and
450 each record will belong to one combination of such.
451 datasetIdentifier : `str` or `None`
452 A string which will be used in creating unique identifier tags.
453 bundle : `MetricMeasurementBundle`
454 The bundle containing metric values to upload.
455 """
456 identifier: str
457 if datasetIdentifier is not None:
458 identifier = datasetIdentifier
459 elif hasattr(bundle, "dataset_identifier") and bundle.dataset_identifier is not None:
460 identifier = bundle.dataset_identifier
461 else:
462 identifier = "Generic"
464 meta["dataset_tag"] = identifier
466 if identifierFields is None:
467 identifierFields = {}
468 for key in IDENTIFIER_KEYS:
469 value = identifierFields.get(key, "")
470 meta[key] = f"{value}"
472 def _prepareBundle(
473 self,
474 bundle: MetricMeasurementBundle,
475 run: str,
476 datasetType: str,
477 timestamp: datetime.datetime | None = None,
478 id: UUID | None = None,
479 identifierFields: Mapping | None = None,
480 datasetIdentifier: str | None = None,
481 extraFields: Mapping | None = None,
482 ) -> tuple[Mapping[str, list[Any]], bool]:
483 """Encode all of the inputs into a format that can be sent to the
484 kafka proxy server.
486 Parameters
487 ----------
488 bundle : `MetricMeasurementBundle`
489 The bundle containing metric values to upload.
490 run : `str`
491 The run name to associate with these metric values. If this bundle
492 is also stored in the butler, this should be the butler run
493 collection the bundle is stored in the butler.
494 datasetType : `str`
495 The dataset type name associated with this
496 `MetricMeasurementBundle`
497 timestamp : `datetime.datetime`, optional
498 The timestamp to be associated with the measurements in the ingress
499 database. If this value is None, timestamp will be set by the run
500 time or current time.
501 id : `UUID`, optional
502 The UUID of the `MetricMeasurementBundle` within the butler. If
503 `None`, a new random UUID will be generated so that each record in
504 Sasquatch will have a unique value.
505 identifierFields: `Mapping`, optional
506 The keys and values in this mapping will be both added as fields
507 in the record, and used in creating a unique tag for the uploaded
508 dataset type. I.e. the timestamp, and the tag will be unique, and
509 each record will belong to one combination of such.
510 datasetIdentifier : `str`, optional
511 A string which will be used in creating unique identifier tags.
512 extraFields: `Mapping`, optional
513 Extra mapping keys and values that will be added as fields to the
514 dispatched record.
516 Returns
517 -------
518 result : `Mapping` of `str` to `list`
519 A mapping of metric name of list of metric measurement records.
520 status : `bool`
521 A status boolean indicating if some records had to be skipped due
522 to a problem parsing the bundle.
523 """
524 if id is None:
525 id = uuid4()
526 sid = str(id)
527 meta: dict[str, Any] = dict()
529 # Add other associated common fields
530 meta["id"] = sid
531 meta["run"] = run
532 meta["dataset_type"] = datasetType
534 # Check to see if the bundle declares a reference package
535 self._handleReferencePackage(meta, bundle)
537 # Handle the various timestamps that could be associated with a record
538 self._handleTimes(meta, bundle, run)
540 # Always use the supplied timestamp if one was passed to use.
541 if timestamp is not None:
542 meta["timestamp"] = timestamp.timestamp()
544 self._handleIdentifier(meta, identifierFields, datasetIdentifier, bundle)
546 # Add in any other fields that were supplied to the function call.
547 if extraFields is not None:
548 meta.update(extraFields)
550 metricRecords: dict[str, list[Any]] = dict()
552 # Record if any records needed skipped
553 resultsTrimmed = False
555 # Look at each of the metrics in the bundle (name, values)
556 for metric, measurements in bundle.items():
557 # Create a list which will contain the records for each measurement
558 # associated with metric.
559 metricRecordList = metricRecords.setdefault(metric, list())
561 record: dict[str, Any] = meta.copy()
563 # loop over each metric measurement within the metric
564 for measurement in measurements:
565 # need to extract any tags, package info, etc
566 note_key = f"{measurement.metric_name.metric}.metric_tags"
567 record["tags"] = dict(measurement.notes.items()).get(note_key, list())
569 # Missing values are replaced by 0 in sasquatch, see RFC-763.
570 name = ""
571 value = 0.0
572 match measurement.json:
573 case {"metric": name, "value": None}:
574 pass
575 case {"metric": name, "value": value}:
576 if math.isnan(value):
577 log.error(
578 "Measurement %s had a value that is a NaN, dispatch will be skipped",
579 measurement,
580 )
581 resultsTrimmed = True
582 continue
583 pass
584 case {"value": _}:
585 log.error("Measurement %s does not contain the key 'metric'", measurement)
586 resultsTrimmed = True
587 continue
588 case {"metric": _}:
589 log.error("Measurement %s does not contain the key 'value'", measurement)
590 resultsTrimmed = True
591 continue
592 record[name] = value
594 metricRecordList.append({"value": record})
595 return metricRecords, resultsTrimmed
597 def dispatch(
598 self,
599 bundle: MetricMeasurementBundle,
600 run: str,
601 datasetType: str,
602 timestamp: datetime.datetime | None = None,
603 id: UUID | None = None,
604 datasetIdentifier: str | None = None,
605 identifierFields: Mapping | None = None,
606 extraFields: Mapping | None = None,
607 ) -> None:
608 """Dispatch a `MetricMeasurementBundle` to Sasquatch.
610 Parameters
611 ----------
612 bundle : `MetricMeasurementBundle`
613 The bundle containing metric values to upload.
614 run : `str`
615 The run name to associate with these metric values. If this bundle
616 is also stored in the butler, this should be the butler run
617 collection the bundle is stored in the butler. This will be used
618 in generating uniqueness constraints in Sasquatch.
619 datasetType : `str`
620 The dataset type name associated with this
621 `MetricMeasurementBundle`.
622 timestamp : `datetime.datetime`, optional
623 The timestamp to be associated with the measurements in the ingress
624 database. If this value is None, timestamp will be set by the run
625 time or current time.
626 id : `UUID`, optional
627 The UUID of the `MetricMeasurementBundle` within the Butler. If
628 `None`, a new random UUID will be generated so that each record in
629 Sasquatch will have a unique value.
630 datasetIdentifier : `str`, optional
631 A string which will be used in creating unique identifier tags. If
632 `None`, a default value will be inserted.
633 identifierFields: `Mapping`, optional
634 The keys and values in this mapping will be both added as fields
635 in the record, and used in creating a unique tag for the uploaded
636 dataset type. I.e. the timestamp, and the tag will be unique, and
637 each record will belong to one combination of such. Examples of
638 entries would be things like visit or tract.
639 extraFields: `Mapping`, optional
640 Extra mapping keys and values that will be added as fields to the
641 dispatched record.
643 Raises
644 ------
645 SasquatchDispatchPartialFailure
646 Raised if there were any errors in dispatching a bundle.
647 """
648 if id is None:
649 id = uuid4()
651 # Prepare the bundle by transforming it to a list of metric records
652 metricRecords, recordsTrimmed = self._prepareBundle(
653 bundle=bundle,
654 run=run,
655 datasetType=datasetType,
656 timestamp=timestamp,
657 id=id,
658 datasetIdentifier=datasetIdentifier,
659 identifierFields=identifierFields,
660 extraFields=extraFields,
661 )
663 headers = {"content-type": "application/vnd.kafka.avro.v2+json"}
664 data: dict[str, Any] = dict()
665 partialUpload = False
666 uploadFailed = []
668 for metric, record in metricRecords.items():
669 # create the kafka topic if it does not already exist
670 if not self._create_topic(metric):
671 log.error("Topic not created, skipping dispatch of %s", metric)
672 continue
673 recordValue = record[0]["value"]
674 # Generate schemas for each record
675 data["value_schema"], schemaTrimmed = self._generateAvroSchema(metric, recordValue)
676 data["records"] = record
678 if schemaTrimmed:
679 partialUpload = True
681 r = requests.post(f"{self.url}/topics/{self.namespace}.{metric}", json=data, headers=headers)
683 if r.status_code == requests.codes.ok:
684 log.debug("Succesfully sent data for metric %s", metric)
685 uploadFailed.append(False)
686 else:
687 log.error(
688 "There was a problem submitting the metric %s: %s, %s", metric, r.status_code, r.json()
689 )
690 uploadFailed.append(True)
691 partialUpload = True
693 # There may be no metrics to try to upload, and thus the uploadFailed
694 # list may be empty, check before issuing failure
695 if len(uploadFailed) > 0 and all(uploadFailed):
696 raise SasquatchDispatchFailure("All records were unable to be uploaded.")
698 if partialUpload or recordsTrimmed:
699 raise SasquatchDispatchPartialFailure("One or more records may not have been uploaded entirely")
701 def dispatchRef(
702 self,
703 bundle: MetricMeasurementBundle,
704 ref: DatasetRef,
705 timestamp: datetime.datetime | None = None,
706 extraFields: Mapping | None = None,
707 datasetIdentifier: str | None = None,
708 ) -> None:
709 """Dispatch a `MetricMeasurementBundle` to Sasquatch with a known
710 `DatasetRef`.
712 Parameters
713 ----------
714 bundle : `MetricMeasurementBundle`
715 The bundle containing metric values to upload.
716 ref : `DatasetRef`
717 The `Butler` dataset ref corresponding to the input
718 `MetricMeasurementBundle`.
719 timestamp : `datetime.datetime`, optional
720 The timestamp to be associated with the measurements in the ingress
721 database. If this value is None, timestamp will be set by the run
722 time or current time.
723 extraFields: `Mapping`, optional
724 Extra mapping keys and values that will be added as fields to the
725 dispatched record if not None.
726 datasetIdentifier : `str`, optional
727 A string which will be used in creating unique identifier tags. If
728 None, a default value will be inserted.
730 Raises
731 ------
732 SasquatchDispatchPartialFailure
733 Raised if there were any errors in dispatching a bundle.
734 """
735 # Parse the relevant info out of the dataset ref.
736 serializedRef = ref.to_simple()
737 id = serializedRef.id
738 if serializedRef.run is None:
739 run = "<unknown>"
740 else:
741 run = serializedRef.run
742 dstype = serializedRef.datasetType
743 datasetType = dstype.name if dstype is not None else ""
744 dataRefMapping = serializedRef.dataId.dataId if serializedRef.dataId else None
746 self.dispatch(
747 bundle,
748 run=run,
749 timestamp=timestamp,
750 datasetType=datasetType,
751 id=id,
752 identifierFields=dataRefMapping,
753 extraFields=extraFields,
754 datasetIdentifier=datasetIdentifier,
755 )