GeezerVisonThe World Wide Web, restored to 800 pixels of glory. |
| Home|About|Gaming cabinet|Archives|How it works|Starter sites |
GeezerVison is showing github.com — datafusion-python/python/datafusion/plan.py at main · apache/datafusion-python Expand file tree/ Copy pathplan.pyMore file actions 581 lines (466 loc) · 22 KB / Copy pathplan.pyFile metadata and controls581 lines (466 loc) · 22 KB 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 # Licensed to the Apache Software Foundation (ASF) under one # or more contributor license agreements. See the NOTICE file # distributed with this work for additional information # regarding copyright ownership. The ASF licenses this file # to you under the Apache License, Version 2.0 (the # "License"); you may not use this file except in compliance # with the License. You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, # software distributed under the License is distributed on an # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. """This module supports physical and logical plans in DataFusion.""" from __future__ import annotations import warnings from typing import TYPE_CHECKING, Any, Literal import datafusion._internal as df_internal if TYPE_CHECKING: import datetime from datafusion.context import SessionContext __all__ = [ "ExecutionPlan", "LogicalPlan", "Metric", "MetricsSet", "PhysicalPartitioning", ] class LogicalPlan: # noqa: PLW1641 """Logical Plan. A `LogicalPlan` is a node in a tree of relational operators (such as Projection or Filter). Represents transforming an input relation (table) to an output relation (table) with a potentially different schema. Plans form a dataflow tree where data flows from leaves up to the root to produce the query result. A `LogicalPlan` can be created by the SQL query planner, the DataFrame API, or programmatically (for example custom query languages). """ def __init__(self, plan: df_internal.LogicalPlan) -> None: """This constructor should not be called by the end user.""" self._raw_plan = plan def to_variant(self) -> Any: """Convert the logical plan into its specific variant.""" return self._raw_plan.to_variant() def inputs(self) -> list[LogicalPlan]: """Returns the list of inputs to the logical plan.""" return [LogicalPlan(p) for p in self._raw_plan.inputs()] def __repr__(self) -> str: """Generate a printable representation of the plan.""" return self._raw_plan.__repr__() def display(self) -> str: """Print the logical plan.""" return self._raw_plan.display() def display_indent(self) -> str: """Print an indented form of the logical plan.""" return self._raw_plan.display_indent() def display_indent_schema(self) -> str: """Print an indented form of the schema for the logical plan.""" return self._raw_plan.display_indent_schema() def display_graphviz(self) -> str: """Print the graph visualization of the logical plan. Returns a formattable structure that produces lines meant for graphical display using the ``DOT`` language. This format can be visualized using software from `graphviz <https://graphviz.org/>`_. """ return self._raw_plan.display_graphviz() @staticmethod def from_bytes(ctx: SessionContext, data: bytes) -> LogicalPlan: """Create a LogicalPlan from serialized protobuf bytes. Decoding routes through the codecs installed on ``ctx`` with :py:meth:`~datafusion.SessionContext.with_logical_extension_codec`. Tables created in memory from record batches are currently not supported. Unlike :py:meth:`datafusion.Expr.from_bytes`, ``ctx`` is required and positional, and there is no fallback to a worker or global context. See Also: :py:meth:`to_bytes`, :py:meth:`ExecutionPlan.from_bytes`, :py:meth:`datafusion.Expr.from_bytes`. """ return LogicalPlan(df_internal.LogicalPlan.from_bytes(ctx.ctx, data)) def to_bytes(self, ctx: SessionContext | None = None) -> bytes: """Convert a LogicalPlan to serialized protobuf bytes. When ``ctx`` is supplied, encoding routes through the codecs installed on it with :py:meth:`~datafusion.SessionContext.with_logical_extension_codec`, so extension codecs see the encode path. With ``ctx=None`` a default codec is used. Tables created in memory from record batches are currently not supported. See Also: :py:meth:`from_bytes`, :py:meth:`ExecutionPlan.to_bytes`, :py:meth:`datafusion.Expr.to_bytes`. """ ctx_arg = ctx.ctx if ctx is not None else None return self._raw_plan.to_bytes(ctx_arg) @staticmethod def from_proto(ctx: SessionContext, data: bytes) -> LogicalPlan: """Deprecated alias for :meth:`from_bytes`.""" warnings.warn( "LogicalPlan.from_proto is deprecated; use from_bytes instead", DeprecationWarning, stacklevel=2, ) return LogicalPlan.from_bytes(ctx, data) def to_proto(self) -> bytes: """Deprecated alias for :meth:`to_bytes`.""" warnings.warn( "LogicalPlan.to_proto is deprecated; use to_bytes instead", DeprecationWarning, stacklevel=2, ) return self.to_bytes() def __eq__(self, other: object) -> bool: """Test equality.""" if not isinstance(other, LogicalPlan): return False return self._raw_plan.__eq__(other._raw_plan) class ExecutionPlan: """Represent nodes in the DataFusion Physical Plan.""" def __init__(self, plan: df_internal.ExecutionPlan) -> None: """This constructor should not be called by the end user.""" self._raw_plan = plan def children(self) -> list[ExecutionPlan]: """Get a list of children ``ExecutionPlan`` that act as inputs to this plan. The returned list will be empty for leaf nodes such as scans, will contain a single value for unary nodes, or two values for binary nodes (such as joins). """ return [ExecutionPlan(e) for e in self._raw_plan.children()] def display(self) -> str: """Print the physical plan.""" return self._raw_plan.display() def display_indent(self) -> str: """Print an indented form of the physical plan.""" return self._raw_plan.display_indent() def __repr__(self) -> str: """Print a string representation of the physical plan.""" return self._raw_plan.__repr__() @property def partition_count(self) -> int: """Returns the number of partitions in the physical plan. Examples: >>> from datafusion import SessionContext >>> ctx = SessionContext() >>> df = ctx.from_pydict({"a": [1, 2, 3]}) >>> df.execution_plan().partition_count 1 """ return self._raw_plan.partition_count @property def output_partitioning(self) -> PhysicalPartitioning: """Returns how this plan's output rows are spread across its partitions. Where :py:attr:`partition_count` gives only the number of partitions, this also reports the scheme, so a caller executing partitions separately can tell whether they are hash-distributed on known keys or merely counted. A plan does not necessarily partition the way it was asked to; see :ref:`checking_partitioning`. Examples: >>> import pyarrow as pa >>> from datafusion import SessionConfig, SessionContext >>> ctx = SessionContext(SessionConfig().with_target_partitions(4)) >>> ctx.register_record_batches("t", [ ... [pa.record_batch({"a": [1, 2, 3]})], ... [pa.record_batch({"a": [4, 5, 6]})], ... ]) >>> ctx.sql("select a from t").execution_plan().output_partitioning UnknownPartitioning(2) A group-by redistributes rows, so the plan reports the keys: >>> grouped = ctx.sql("select a, count(*) from t group by a") >>> partitioning = grouped.execution_plan().output_partitioning >>> partitioning.scheme 'Hash' >>> partitioning.partition_count 4 """ return PhysicalPartitioning(self._raw_plan.output_partitioning) @staticmethod def from_bytes(ctx: SessionContext, data: bytes) -> ExecutionPlan: """Create an ExecutionPlan from serialized protobuf bytes. Decoding routes through the codecs installed on ``ctx`` with :py:meth:`~datafusion.SessionContext.with_physical_extension_codec`. Unlike :py:meth:`datafusion.Expr.from_bytes`, ``ctx`` is required and positional, and there is no fallback to a worker or global context. ``ctx`` need share nothing with the session that encoded the plan: a scan over a table registered from record batches decodes here, because the batches travel inside the encoded scan. See Also: :py:meth:`to_bytes`, :py:meth:`LogicalPlan.from_bytes`. """ return ExecutionPlan(df_internal.ExecutionPlan.from_bytes(ctx.ctx, data)) def to_bytes(self, ctx: SessionContext | None = None) -> bytes: """Convert an ExecutionPlan into serialized protobuf bytes. When ``ctx`` is supplied, encoding routes through the codecs installed on it with :py:meth:`~datafusion.SessionContext.with_physical_extension_codec`. Unlike :py:meth:`LogicalPlan.to_bytes`, a plan reading a table registered from record batches does round-trip: the batches travel inside the encoded scan. Round-tripping through this method and :py:meth:`from_bytes` is how an extension library checks that its own codec claimed its nodes, rather than a codec installed earlier in the chain — see :ref:`extension_codec_order`. See Also: :py:meth:`from_bytes`, :py:meth:`LogicalPlan.to_bytes`. """ ctx_arg = ctx.ctx if ctx is not None else None return self._raw_plan.to_bytes(ctx_arg) @staticmethod def from_proto(ctx: SessionContext, data: bytes) -> ExecutionPlan: """Deprecated alias for :meth:`from_bytes`.""" warnings.warn( "ExecutionPlan.from_proto is deprecated; use from_bytes instead", DeprecationWarning, stacklevel=2, ) return ExecutionPlan.from_bytes(ctx, data) def to_proto(self) -> bytes: """Deprecated alias for :meth:`to_bytes`.""" warnings.warn( "ExecutionPlan.to_proto is deprecated; use to_bytes instead", DeprecationWarning, stacklevel=2, ) return self.to_bytes() def metrics(self) -> MetricsSet | None: """Return metrics for this plan node, or None if this plan has no MetricsSet. Some operators (e.g. DataSourceExec) eagerly initialize a MetricsSet when the plan is created, so this may return a set even before execution. Metric *values* (such as ``output_rows``) are only meaningful after the DataFrame has been executed. """ raw = self._raw_plan.metrics() if raw is None: return None return MetricsSet(raw) def collect_metrics(self) -> list[tuple[str, MetricsSet]]: """Return runtime statistics for each step of the query execution. DataFusion executes a query as a pipeline of operators — for example a data source scan, followed by a filter, followed by a projection. After the DataFrame has been executed (via :py:meth:`~datafusion.DataFrame.collect`, :py:meth:`~datafusion.DataFrame.execute_stream`, etc.), each operator records statistics such as how many rows it produced and how much CPU time it consumed. Each entry in the returned list corresponds to one operator that recorded metrics. The first element of the tuple is the operator's description string — the same text shown by :py:meth:`display_indent` — which identifies both the operator type and its key parameters, for example ``"FilterExec: column1@0 > 1"`` or ``"DataSourceExec: partitions=1"``. Returns: A list of ``(description, MetricsSet)`` tuples ordered from the outermost operator (top of the execution tree) down to the data-source leaves. Only operators that recorded at least one metric are included. Returns an empty list if called before the DataFrame has been executed. """ result: list[tuple[str, MetricsSet]] = [] def _walk(node: ExecutionPlan) -> None: ms = node.metrics() if ms is not None: result.append((node.display(), ms)) for child in node.children(): _walk(child) _walk(self) return result class PhysicalPartitioning: """How a physical plan's output rows are spread across its partitions. Returned by :py:attr:`ExecutionPlan.output_partitioning`. This is the partitioning a built plan *has*. Distinct from :py:class:`datafusion.expr.Partitioning`, the *logical* partitioning a ``Repartition`` node records and hands back from ``partitioning_scheme()`` — a request, which the plan need not honour. See :ref:`checking_partitioning`. """ def __init__(self, partitioning: df_internal.PhysicalPartitioning) -> None: """This constructor should not be called by the end user.""" self._raw_partitioning = partitioning @property def scheme( self, ) -> Literal["RoundRobinBatch", "Hash", "Range", "UnknownPartitioning"]: """Which partitioning scheme this is. ``"UnknownPartitioning"`` means the plan knows how many partitions it has but nothing about how rows are distributed between them, which is the usual case for a file scan. ``"RoundRobinBatch"`` and ``"Hash"`` come from a repartition the optimizer inserted. ``"Range"`` only appears on a plan this package did not build; see :ref:`checking_partitioning`. Examples: >>> from datafusion import SessionContext >>> ctx = SessionContext() >>> df = ctx.from_pydict({"a": [1, 2, 3]}) >>> df.execution_plan().output_partitioning.scheme 'UnknownPartitioning' """ return self._raw_partitioning.scheme @property def partition_count(self) -> int: """The number of partitions. Examples: >>> from datafusion import SessionContext >>> ctx = SessionContext() >>> df = ctx.from_pydict({"a": [1, 2, 3]}) >>> df.execution_plan().output_partitioning.partition_count 1 """ return self._raw_partitioning.partition_count @property def hash_expressions(self) -> list[str] | None: """The expressions rows are hashed on, or ``None`` for other schemes. Physical expressions have no Python representation, so these are returned in their displayed form. ``None`` covers ``"Range"`` as well, whose ordering and split points this class does not expose. Examples: >>> import pyarrow as pa >>> from datafusion import SessionConfig, SessionContext >>> ctx = SessionContext(SessionConfig().with_target_partitions(4)) >>> ctx.register_record_batches("t", [ ... [pa.record_batch({"a": [1, 2, 3]})], ... [pa.record_batch({"a": [4, 5, 6]})], ... ]) >>> scan = ctx.sql("select a from t").execution_plan() >>> scan.output_partitioning.hash_expressions is None True >>> grouped = ctx.sql("select a, count(*) from t group by a") >>> grouped.execution_plan().output_partitioning.hash_expressions ['a@0'] """ return self._raw_partitioning.hash_expressions def __repr__(self) -> str: """Print a string representation of the partitioning. Examples: >>> from datafusion import SessionContext >>> ctx = SessionContext() >>> df = ctx.from_pydict({"a": [1, 2, 3]}) >>> repr(df.execution_plan().output_partitioning) 'UnknownPartitioning(1)' """ return self._raw_partitioning.__repr__() def _key(self) -> tuple[str, int, tuple[str, ...] | None]: exprs = self.hash_expressions return (self.scheme, self.partition_count, tuple(exprs) if exprs else None) def __eq__(self, other: object) -> bool: """Compare two partitionings by scheme, count and hash expressions. Equality is structural, and does not mirror DataFusion's own comparison of the underlying type, under which two ``UnknownPartitioning`` values of the same width are unequal. Examples: >>> from datafusion import SessionContext >>> ctx = SessionContext() >>> a = ctx.from_pydict({"a": [1, 2, 3]}).execution_plan() >>> b = ctx.from_pydict({"b": [4, 5, 6]}).execution_plan() >>> a.output_partitioning == b.output_partitioning True >>> a.output_partitioning == "UnknownPartitioning(1)" False """ if not isinstance(other, PhysicalPartitioning): return NotImplemented return self._key() == other._key() def __hash__(self) -> int: """Hash the partitioning, consistently with :py:meth:`__eq__`. Examples: >>> from datafusion import SessionContext >>> ctx = SessionContext() >>> a = ctx.from_pydict({"a": [1, 2, 3]}).execution_plan() >>> b = ctx.from_pydict({"b": [4, 5, 6]}).execution_plan() >>> len({a.output_partitioning, b.output_partitioning}) 1 """ return hash(self._key()) class MetricsSet: """A set of metrics for a single execution plan operator. A physical plan operator runs independently across one or more partitions. :py:meth:`metrics` returns the raw per-partition :py:class:`Metric` objects. The convenience properties (:py:attr:`output_rows`, :py:attr:`elapsed_compute`, etc.) automatically sum the named metric across *all* partitions, giving a single aggregate value for the operator as a whole. """ def __init__(self, raw: df_internal.MetricsSet) -> None: """This constructor should not be called by the end user.""" self._raw = raw def metrics(self) -> list[Metric]: """Return all individual metrics in this set.""" return [Metric(m) for m in self._raw.metrics()] @property def output_rows(self) -> int | None: """Sum of output_rows across all partitions.""" return self._raw.output_rows() @property def elapsed_compute(self) -> int | None: """Total CPU time (in nanoseconds) spent inside this operator's execute loop. Summed across all partitions. Returns ``None`` if no ``elapsed_compute`` metric was recorded. """ return self._raw.elapsed_compute() @property def spill_count(self) -> int | None: """Number of times this operator spilled data to disk due to memory pressure. This is a count of spill events, not a byte count. Summed across all partitions. Returns ``None`` if no ``spill_count`` metric was recorded. """ return self._raw.spill_count() @property def spilled_bytes(self) -> int | None: """Sum of spilled_bytes across all partitions.""" return self._raw.spilled_bytes() @property def spilled_rows(self) -> int | None: """Sum of spilled_rows across all partitions.""" return self._raw.spilled_rows() def sum_by_name(self, name: str) -> int | None: """Sum the named metric across all partitions. Useful for accessing any metric not exposed as a first-class property. Returns ``None`` if no metric with the given name was recorded. Args: name: The metric name, e.g. ``"output_rows"`` or ``"elapsed_compute"``. """ return self._raw.sum_by_name(name) def __repr__(self) -> str: """Return a string representation of the metrics set.""" return repr(self._raw) class Metric: """A single execution metric with name, value, partition, and labels.""" def __init__(self, raw: df_internal.Metric) -> None: """This constructor should not be called by the end user.""" self._raw = raw @property def name(self) -> str: """The name of this metric (e.g. ``output_rows``).""" return self._raw.name @property def value(self) -> int | datetime.datetime | None: """The value of this metric. Returns an ``int`` for counters, gauges, and time-based metrics (nanoseconds), a :py:class:`~datetime.datetime` (UTC) for ``start_timestamp`` / ``end_timestamp`` metrics, or ``None`` when the value has not been set or is not representable. """ return self._raw.value @property def value_as_datetime(self) -> datetime.datetime | None: """The value as a UTC :py:class:`~datetime.datetime` for timestamp metrics. Returns ``None`` for all non-timestamp metrics and for timestamp metrics whose value has not been set (e.g. before execution). """ return self._raw.value_as_datetime @property def partition(self) -> int | None: """The 0-based partition index this metric applies to. Returns ``None`` for metrics that are not partition-specific (i.e. they apply globally across all partitions of the operator). """ return self._raw.partition def labels(self) -> dict[str, str]: """Return the labels associated with this metric. Labels provide additional context for a metric. For example:: metric.labels() # {'output_type': 'final'} """ return self._raw.labels() def __repr__(self) -> str: """Return a string representation of the metric.""" return repr(self._raw) |
| You are visitor 01402520 — GeezerVison does not store the pages you look up. Best viewed with Netscape Navigator 4.7 or Internet Explorer 6 at 800×600. No cookies required. |