GeezerVison is showing github.com — feat: accept distinct kwarg on sum and avg (#1556) · apache/datafusion-python@0840763
File treeExpand file treeCollapse file tree Open diff view settings .ai/skills/check-upstream
Expand file treeCollapse file tree Open diff view settings Collapse file Copy file name to clipboardExpand all lines: .ai/skills/check-upstream/SKILL.md +8-1Lines changed: 8 additions & 1 deletion - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -88,11 +88,18 @@ The user may specify an area via `$ARGUMENTS`. If no area is specified or "all" | 88 | 88 | - Python API: `python/datafusion/functions.py` (aggregate functions are mixed in with scalar functions) | 89 | 89 | - Rust bindings: `crates/core/src/functions.rs` | 90 | 90 |
| | 91 | +**Evaluated and not requiring separate Python exposure:** | | 92 | +- `count_distinct` — covered by `count(expr, distinct=True)`. Both forms call | | 93 | + `count_udaf` with `distinct: bool = true` and produce the same logical plan. | | 94 | +- `sum_distinct` — covered by `sum(expr, distinct=True)`. | | 95 | +- `avg_distinct` — covered by `avg(expr, distinct=True)`. | | 96 | + | 91 | 97 | **How to check:** | 92 | 98 | 1. Fetch the upstream aggregate function documentation page | 93 | 99 | 2. Compare against aggregate functions in `python/datafusion/functions.py` (check `__all__` list and function definitions) | 94 | 100 | 3. A function is covered if it exists in the Python API, even if it aliases another function's Rust binding | 95 | | -4. Report only functions missing from the Python API | | 101 | +4. Check against the "evaluated and not requiring exposure" list before flagging as a gap | | 102 | +5. Report only functions missing from the Python API | 96 | 103 |
| 97 | 104 | ### 3. Window Functions | 98 | 105 |
| |
Collapse file Copy file name to clipboardExpand all lines: docs/source/user-guide/upgrade-guides.rst +20Lines changed: 20 additions & 0 deletions - Display the source diff
- Display the rich diff
| Original file line number | Diff line number | Diff line change |
|---|
@@ -45,6 +45,26 @@ After: | 45 | 45 | config = SessionConfig().set("datafusion.execution.batch_size", "4096") | 46 | 46 | ctx = SessionContext(config) | 47 | 47 |
| | 48 | +The aggregate functions :py:func:`~datafusion.functions.sum` and | | 49 | +:py:func:`~datafusion.functions.avg` now accept a ``distinct`` argument, matching | | 50 | +the other aggregate functions. ``distinct`` is inserted *before* ``filter`` in the | | 51 | +argument list, so any code that passed ``filter`` positionally must be updated to | | 52 | +pass it as a keyword argument. The types are distinct so a type checker should flag this. | | 53 | + | | 54 | +Before: | | 55 | + | | 56 | +.. code-block:: python | | 57 | +
| | 58 | + f.sum(column("a"), my_filter) | | 59 | + f.avg(column("a"), my_filter) | | 60 | +
| | 61 | +Now: | | 62 | + | | 63 | +.. code-block:: python | | 64 | +
| | 65 | + f.sum(column("a"), filter=my_filter) | | 66 | + f.avg(column("a"), filter=my_filter) | | 67 | +
| 48 | 68 | DataFusion 53.0.0 | 49 | 69 | ----------------- | 50 | 70 |
| |
Collapse file Copy file name to clipboardExpand all lines: python/datafusion/functions.py +24-4Lines changed: 24 additions & 4 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -4563,17 +4563,19 @@ def grouping( | 4563 | 4563 |
| 4564 | 4564 | def avg( | 4565 | 4565 | expression: Expr, | | 4566 | + distinct: bool = False, | 4566 | 4567 | filter: Expr | None = None, | 4567 | 4568 | ) -> Expr: | 4568 | 4569 | """Returns the average value. | 4569 | 4570 |
| 4570 | 4571 | This aggregate function expects a numeric expression and will return a float. | 4571 | 4572 |
| 4572 | 4573 | If using the builder functions described in ref:`_aggregation` this function ignores | 4573 | | - the options ``order_by``, ``null_treatment``, and ``distinct``. | | 4574 | + the options ``order_by`` and ``null_treatment``. | 4574 | 4575 |
| 4575 | 4576 | Args: | 4576 | 4577 | expression: Values to combine into an array | | 4578 | + distinct: If True, duplicate values are removed before averaging. | 4577 | 4579 | filter: If provided, only compute against rows for which the filter is True | 4578 | 4580 |
| 4579 | 4581 | Examples: | @@ -4593,9 +4595,17 @@ def avg( | 4593 | 4595 | ... ).alias("v")]) | 4594 | 4596 | >>> result.collect_column("v")[0].as_py() | 4595 | 4597 | 2.5 | | 4598 | +
| | 4599 | + >>> df = ctx.from_pydict({"a": [1.0, 1.0, 2.0, 3.0]}) | | 4600 | + >>> result = df.aggregate( | | 4601 | + ... [], [dfn.functions.avg( | | 4602 | + ... dfn.col("a"), distinct=True, | | 4603 | + ... ).alias("v")]) | | 4604 | + >>> result.collect_column("v")[0].as_py() | | 4605 | + 2.0 | 4596 | 4606 | """ | 4597 | 4607 | filter_raw = filter.expr if filter is not None else None | 4598 | | - return Expr(f.avg(expression.expr, filter=filter_raw)) | | 4608 | + return Expr(f.avg(expression.expr, distinct=distinct, filter=filter_raw)) | 4599 | 4609 |
| 4600 | 4610 |
| 4601 | 4611 | def corr(value_y: Expr, value_x: Expr, filter: Expr | None = None) -> Expr: | @@ -4880,17 +4890,19 @@ def min(expression: Expr, filter: Expr | None = None) -> Expr: | 4880 | 4890 |
| 4881 | 4891 | def sum( | 4882 | 4892 | expression: Expr, | | 4893 | + distinct: bool = False, | 4883 | 4894 | filter: Expr | None = None, | 4884 | 4895 | ) -> Expr: | 4885 | 4896 | """Computes the sum of a set of numbers. | 4886 | 4897 |
| 4887 | 4898 | This aggregate function expects a numeric expression. | 4888 | 4899 |
| 4889 | 4900 | If using the builder functions described in ref:`_aggregation` this function ignores | 4890 | | - the options ``order_by``, ``null_treatment``, and ``distinct``. | | 4901 | + the options ``order_by`` and ``null_treatment``. | 4891 | 4902 |
| 4892 | 4903 | Args: | 4893 | 4904 | expression: Values to combine into an array | | 4905 | + distinct: If True, duplicate values are removed before summing. | 4894 | 4906 | filter: If provided, only compute against rows for which the filter is True | 4895 | 4907 |
| 4896 | 4908 | Examples: | @@ -4910,9 +4922,17 @@ def sum( | 4910 | 4922 | ... ).alias("v")]) | 4911 | 4923 | >>> result.collect_column("v")[0].as_py() | 4912 | 4924 | 5 | | 4925 | +
| | 4926 | + >>> df = ctx.from_pydict({"a": [1, 1, 2, 3]}) | | 4927 | + >>> result = df.aggregate( | | 4928 | + ... [], [dfn.functions.sum( | | 4929 | + ... dfn.col("a"), distinct=True, | | 4930 | + ... ).alias("v")]) | | 4931 | + >>> result.collect_column("v")[0].as_py() | | 4932 | + 6 | 4913 | 4933 | """ | 4914 | 4934 | filter_raw = filter.expr if filter is not None else None | 4915 | | - return Expr(f.sum(expression.expr, filter=filter_raw)) | | 4935 | + return Expr(f.sum(expression.expr, distinct=distinct, filter=filter_raw)) | 4916 | 4936 |
| 4917 | 4937 |
| 4918 | 4938 | def stddev(expression: Expr, filter: Expr | None = None) -> Expr: | |
Collapse file Copy file name to clipboardExpand all lines: python/tests/test_aggregation.py +2Lines changed: 2 additions & 0 deletions | Original file line number | Diff line number | Diff line change |
|---|
@@ -192,7 +192,9 @@ def test_aggregation_stats(df, agg_expr, calc_expected): | 192 | 192 | False, | 193 | 193 | ), | 194 | 194 | (f.avg(column("b"), filter=column("a") != lit(1)), pa.array([5.0]), False), | | 195 | + (f.avg(column("b"), distinct=True), pa.array([5.0]), False), | 195 | 196 | (f.sum(column("b"), filter=column("a") != lit(1)), pa.array([10]), False), | | 197 | + (f.sum(column("b"), distinct=True), pa.array([10]), False), | 196 | 198 | (f.count(column("b"), distinct=True), pa.array([2]), False), | 197 | 199 | (f.count(column("b"), filter=column("a") != 3), pa.array([2]), False), | 198 | 200 | (f.count(), pa.array([3]), False), | |
|