Skip to content

[SPARK-59170][SQL] Align SQL function groupings across APIs and documentation #58468

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
zhengruifeng wants to merge 4 commits into
base: master
Choose a base branch
from
Draft

[SPARK-59170][SQL] Align SQL function groupings across APIs and documentation #58468

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
873711a
[SPARK-59170][SQL] Align SQL function groupings across APIs and docum…
zhengruifeng Sep 2, 2026
5df40d2
[SPARK-59170][SQL] Align Scala and Python function sections
zhengruifeng Sep 2, 2026
6eba26c
Revert "[SPARK-59170][SQL] Align Scala and Python function sections"
zhengruifeng Sep 2, 2026
9025152
[SPARK-59170][SQL] Remove misleading function section headings
zhengruifeng Sep 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 2 additions & 0 deletions python/docs/source/reference/pyspark.sql/functions.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -454,13 +454,15 @@ Aggregate Functions
bit_and
bit_or
bit_xor
bitmap_and_agg
bitmap_construct_agg
bitmap_or_agg
bitmap_xor_agg
bool_and
bool_or
collect_list
collect_set
collect_union
corr
count
count_distinct
Expand Down
4 changes: 2 additions & 2 deletions python/pyspark/sql/functions/__init__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@
from pyspark.sql.functions.builtin import * # noqa: F403

__all__ = [ # noqa: F405
# Normal functions
# Normal Functions
"broadcast",
"call_function",
"col",
Expand Down Expand Up @@ -620,7 +620,7 @@
"vector_normalize",
"vector_avg",
"vector_sum",
# Call Functions
# UDF, UDTF and UDT
"call_udf",
"pandas_udf",
"udaf",
Expand Down
38 changes: 2 additions & 36 deletions python/pyspark/sql/functions/builtin.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -109,6 +109,8 @@
# even though there might be few exceptions for legacy or inevitable reasons.
# If you are fixing other language APIs together, also please note that Scala side is not the case
# since it requires making every single overridden definition.
# Public function groups are defined by pyspark.sql.functions.__all__ and mirrored in the API
# reference.


def _get_jvm_function(name: str, sc: "SparkContext") -> Callable:
Expand Down Expand Up @@ -9213,9 +9215,6 @@ def factorial(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("factorial", col)


# --------------- Window functions ------------------------


@_try_remote_functions
def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column:
"""
Expand Down Expand Up @@ -9816,9 +9815,6 @@ def ntile(n: int) -> Column:
return _invoke_function("ntile", int(_enum_to_value(n)))


# ---------------------- Date/Timestamp functions ------------------------------


@_try_remote_functions
def curdate() -> Column:
"""
Expand Down Expand Up @@ -14551,9 +14547,6 @@ def to_timestamp_ntz(
return _invoke_function_over_columns("to_timestamp_ntz", timestamp)


# ---------------------------- misc functions ----------------------------------


@_try_remote_functions
def current_catalog() -> Column:
"""Returns the current catalog.
Expand Down Expand Up @@ -15237,9 +15230,6 @@ def raise_error(errMsg: Union[Column, str]) -> Column:
return _invoke_function_over_columns("raise_error", lit(errMsg))


# ---------------------- String/Binary functions ------------------------------


@_try_remote_functions
def upper(col: "ColumnOrName") -> Column:
"""
Expand Down Expand Up @@ -19939,9 +19929,6 @@ def quote(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("quote", col)


# ---------------------- Collection functions ------------------------------


@overload
def create_map(*cols: "ColumnOrName") -> Column: ...

Expand Down Expand Up @@ -26848,9 +26835,6 @@ def str_to_map(
return _invoke_function_over_columns("str_to_map", text, pairDelim, keyValueDelim)


# ---------------------- Partition transform functions --------------------------------


@_try_remote_functions
def years(col: "ColumnOrName") -> Column:
"""
Expand Down Expand Up @@ -28910,9 +28894,6 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column:
return partitioning.bucket(numBuckets, col)


# Geospatial ST Functions


@_try_remote_functions
def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None) -> Column:
"""Returns the input GEOGRAPHY or GEOMETRY value in WKB format.
Expand Down Expand Up @@ -29080,9 +29061,6 @@ def st_srid(geo: "ColumnOrName") -> Column:
return _invoke_function_over_columns("st_srid", geo)


# Call Functions


@_try_remote_functions
def call_udf(udfName: str, *cols: "ColumnOrName") -> Column:
"""
Expand Down Expand Up @@ -29369,9 +29347,6 @@ def wrap_udt(col: "ColumnOrName", udt: "Union[UserDefinedType, Column]") -> Colu
return _invoke_function("wrap_udt", _to_java_column(col), _to_java_column(udt_col))


# ---------------------- Datasketch functions ------------------------------


@_try_remote_functions
def hll_sketch_agg(
col: "ColumnOrName",
Expand Down Expand Up @@ -31915,9 +31890,6 @@ def tuple_union_theta_integer(
return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode)


# ---------------------- Predicates functions ------------------------------


@_try_remote_functions
def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column:
"""
Expand Down Expand Up @@ -33462,9 +33434,6 @@ def bitmap_xor_agg(col: "ColumnOrName") -> Column:
return _invoke_function_over_columns("bitmap_xor_agg", col)


# ---------------------------- User Defined Function ----------------------------------


def udaf(agg: "Aggregator") -> "UserDefinedFunctionLike":
"""Turn an :class:`~pyspark.sql.aggregator.Aggregator` instance into a callable usable in
``groupBy().agg(...)`` (and as a window function), the Python counterpart of Scala's
Expand Down Expand Up @@ -34038,9 +34007,6 @@ def arrow_udtf(
return _create_pyarrow_udtf(cls=cls, returnType=returnType)


# ---------------------- Vector Functions ----------------------


@_try_remote_functions
def vector_cosine_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column:
"""Returns the cosine similarity between two float vectors.
Expand Down
Loading