From 873711a3f825dac8a4ecbc68a2b1f974874feb58 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 2 Sep 2026 08:50:36 +0000 Subject: [PATCH 1/4] [SPARK-59170][SQL] Align SQL function groupings across APIs and documentation --- .../reference/pyspark.sql/functions.rst | 2 + python/pyspark/sql/functions/__init__.py | 4 +- python/pyspark/sql/functions/builtin.py | 3 + .../org/apache/spark/sql/functions.scala | 37 +- .../catalyst/analysis/FunctionRegistry.scala | 749 +++++++++--------- 5 files changed, 404 insertions(+), 391 deletions(-) diff --git a/python/docs/source/reference/pyspark.sql/functions.rst b/python/docs/source/reference/pyspark.sql/functions.rst index 94c5a9a87e13a..379393e26e57d 100644 --- a/python/docs/source/reference/pyspark.sql/functions.rst +++ b/python/docs/source/reference/pyspark.sql/functions.rst @@ -454,6 +454,7 @@ Aggregate Functions bit_and bit_or bit_xor + bitmap_and_agg bitmap_construct_agg bitmap_or_agg bitmap_xor_agg @@ -461,6 +462,7 @@ Aggregate Functions bool_or collect_list collect_set + collect_union corr count count_distinct diff --git a/python/pyspark/sql/functions/__init__.py b/python/pyspark/sql/functions/__init__.py index 861f5902f977a..36750573d5846 100644 --- a/python/pyspark/sql/functions/__init__.py +++ b/python/pyspark/sql/functions/__init__.py @@ -21,7 +21,7 @@ from pyspark.sql.functions.builtin import * # noqa: F403 __all__ = [ # noqa: F405 - # Normal functions + # Normal Functions "broadcast", "call_function", "col", @@ -620,7 +620,7 @@ "vector_normalize", "vector_avg", "vector_sum", - # Call Functions + # UDF, UDTF and UDT "call_udf", "pandas_udf", "udaf", diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index e54927c36e651..54a55f91cd9fb 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -109,6 +109,9 @@ # even though there might be few exceptions for legacy or inevitable reasons. # If you are fixing other language APIs together, also please note that Scala side is not the case # since it requires making every single overridden definition. +# Public function groups are defined by pyspark.sql.functions.__all__ and mirrored in the API +# reference. +# Section headings in this implementation file are only navigation aids. def _get_jvm_function(name: str, sc: "SparkContext") -> Callable: diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 307a82e5a44a6..efa69f4154ce6 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -53,33 +53,33 @@ import org.apache.spark.util.SparkClassUtils * only `Column` but also other types such as a native string. The other variants currently exist * for historical reasons. * - * @groupname udf_funcs UDF, UDAF and UDT - * @groupname agg_funcs Aggregate functions - * @groupname datetime_funcs Date and Timestamp functions - * @groupname sort_funcs Sort functions * @groupname normal_funcs Normal functions + * @groupname conditional_funcs Conditional functions + * @groupname predicate_funcs Predicate functions + * @groupname sort_funcs Sort functions * @groupname math_funcs Mathematical functions + * @groupname string_funcs String functions * @groupname bitwise_funcs Bitwise functions - * @groupname predicate_funcs Predicate functions - * @groupname conditional_funcs Conditional functions + * @groupname datetime_funcs Date and Timestamp functions * @groupname hash_funcs Hash functions - * @groupname misc_funcs Misc functions - * @groupname sketch_funcs Datasketch functions - * @groupname window_funcs Window functions - * @groupname generator_funcs Generator functions - * @groupname string_funcs String functions * @groupname collection_funcs Collection functions * @groupname array_funcs Array functions - * @groupname map_funcs Map functions * @groupname struct_funcs Struct functions - * @groupname st_funcs ST geospatial functions + * @groupname map_funcs Map functions + * @groupname agg_funcs Aggregate functions + * @groupname window_funcs Window functions + * @groupname generator_funcs Generator functions + * @groupname partition_transforms Partition transform functions * @groupname csv_funcs CSV functions * @groupname json_funcs JSON functions * @groupname variant_funcs VARIANT functions - * @groupname vector_funcs Vector functions * @groupname xml_funcs XML functions * @groupname url_funcs URL functions - * @groupname partition_transforms Partition transform functions + * @groupname misc_funcs Misc functions + * @groupname sketch_funcs Datasketch functions + * @groupname st_funcs ST geospatial functions + * @groupname vector_funcs Vector functions + * @groupname udf_funcs UDF, UDAF and UDT * @groupname Ungrouped Support functions for DataFrames * @since 1.3.0 */ @@ -88,6 +88,9 @@ import org.apache.spark.util.SparkClassUtils object functions { // scalastyle:on + // Function groups are defined by the @group tags above each function and the corresponding + // @groupname declarations. Section headings in this implementation file are navigation aids. + /** * Returns a [[Column]] based on the given column name. * @@ -4686,7 +4689,7 @@ object functions { * * @param e * the value to compute the mean of. A column that evaluates to a numeric or interval. - * @group math_funcs + * @group agg_funcs * @since 3.5.0 * @return * Returns a column that evaluates to a double. @@ -4757,7 +4760,7 @@ object functions { * * @param e * the value to compute the sum of. A column that evaluates to a numeric or interval. - * @group math_funcs + * @group agg_funcs * @since 3.5.0 * @return * Returns a column that evaluates to a numeric. diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala index 147a980eeb098..0ed6cee205da7 100644 --- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala +++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala @@ -405,40 +405,55 @@ object FunctionRegistry { // anymore. See `AesEncrypt`/`AesDecrypt` as an example. private type FunctionRegistryEntry = (String, (ExpressionInfo, FunctionBuilder)) - private def miscNonAggregateExpressions: Seq[FunctionRegistryEntry] = Seq( - // misc non-aggregate functions - expression[Abs]("abs"), + private def conditionalExpressions: Seq[FunctionRegistryEntry] = Seq( + // conditional functions expression[Coalesce]("coalesce"), - expressionBuilder("explode", ExplodeExpressionBuilder), - expressionGeneratorBuilderOuter("explode_outer", ExplodeExpressionBuilder), - expression[Greatest]("greatest"), expression[If]("if"), - expressionBuilder("inline", InlineExpressionBuilder), - expressionGeneratorBuilderOuter("inline_outer", InlineExpressionBuilder), - expression[IsNaN]("isnan"), expression[Nvl]("ifnull", setAlias = true), - expression[IsNull]("isnull"), - expression[IsNotNull]("isnotnull"), - expression[Least]("least"), expression[NaNvl]("nanvl"), expression[NullIf]("nullif"), expression[NullIfZero]("nullifzero"), expression[Nvl]("nvl"), expression[Nvl2]("nvl2"), - expressionBuilder("posexplode", PosExplodeExpressionBuilder), - expressionGeneratorBuilderOuter("posexplode_outer", PosExplodeExpressionBuilder), - expression[Rand]("rand"), - expression[Rand]("random", true, Some("3.0.0")), - expression[Randn]("randn"), - expression[RandStr]("randstr"), - expression[Stack]("stack"), - expression[Uniform]("uniform"), expression[ZeroIfNull]("zeroifnull"), - CaseWhen.registryEntry + CaseWhen.registryEntry, + expression[Between]("between") + ) + + private def predicateExpressions: Seq[FunctionRegistryEntry] = Seq( + // predicate functions + expression[IsNaN]("isnan"), + expression[IsNull]("isnull"), + expression[IsNotNull]("isnotnull"), + expression[Like]("like"), + expression[ILike]("ilike"), + expression[RLike]("rlike"), + expression[RLike]("regexp_like", true, Some("3.2.0")), + expression[RLike]("regexp", true, Some("3.2.0")), + expression[EqualNull]("equal_null"), + expression[And]("and"), + expression[In]("in"), + expression[Not]("not"), + expression[Or]("or"), + expression[EqualNullSafe]("<=>"), + expression[EqualTo]("="), + expression[EqualTo]("=="), + expression[GreaterThan](">"), + expression[GreaterThanOrEqual](">="), + expression[LessThan]("<"), + expression[LessThanOrEqual]("<="), + expression[Not]("!") ) private def mathExpressions: Seq[FunctionRegistryEntry] = Seq( // math functions + expression[Abs]("abs"), + expression[Greatest]("greatest"), + expression[Least]("least"), + expression[Rand]("rand"), + expression[Rand]("random", true, Some("3.0.0")), + expression[Randn]("randn"), + expression[Uniform]("uniform"), expression[Acos]("acos"), expression[Acosh]("acosh"), expression[Asin]("asin"), @@ -479,142 +494,34 @@ object FunctionRegistry { expression[Rint]("rint"), expression[Round]("round"), expression[Truncate]("truncate"), - expression[ShiftLeft]("shiftleft"), - expression[ShiftRight]("shiftright"), - expression[ShiftRightUnsigned]("shiftrightunsigned"), expression[Signum]("sign", true), expression[Signum]("signum"), expression[Sin]("sin"), expression[Csc]("csc"), expression[Sinh]("sinh"), - expression[StringToMap]("str_to_map"), expression[Sqrt]("sqrt"), expression[Tan]("tan"), expression[Cot]("cot"), expression[Tanh]("tanh"), expression[WidthBucket]("width_bucket"), - expression[Add]("+"), expression[Subtract]("-"), expression[Multiply]("*"), expression[Divide]("/"), expression[IntegralDivide]("div"), - expression[Remainder]("%") - ) - - private def tryExpressions: Seq[FunctionRegistryEntry] = Seq( - // "try_*" function which always return Null instead of runtime error. + expression[Remainder]("%"), expression[TryAdd]("try_add"), expression[TryDivide]("try_divide"), expression[TryMod]("try_mod"), expression[TrySubtract]("try_subtract"), expression[TryMultiply]("try_multiply"), - expression[TryElementAt]("try_element_at"), - expressionBuilder("try_avg", TryAverageExpressionBuilder, setAlias = true), - expressionBuilder("try_sum", TrySumExpressionBuilder, setAlias = true), - expression[TryToBinary]("try_to_binary"), - expressionBuilder("try_to_timestamp", TryToTimestampExpressionBuilder, setAlias = true), - expressionBuilder("try_to_date", TryToDateExpressionBuilder, setAlias = true), - expressionBuilder("try_to_time", TryToTimeExpressionBuilder, setAlias = true), - expression[TryAesDecrypt]("try_aes_decrypt"), - expression[TryReflect]("try_reflect"), - expression[TryUrlDecode]("try_url_decode"), - expression[TryMakeInterval]("try_make_interval") - ) - - private def aggregateExpressions: Seq[FunctionRegistryEntry] = Seq( - // aggregate functions - expression[HyperLogLogPlusPlus]("approx_count_distinct"), - expression[Average]("avg"), - expression[Corr]("corr"), - expression[Count]("count"), - expression[CountIf]("count_if"), - expression[CovPopulation]("covar_pop"), - expression[CovSample]("covar_samp"), - expression[First]("first"), - expression[First]("first_value", true), - expression[AnyValue]("any_value"), - expression[Kurtosis]("kurtosis"), - expression[Last]("last"), - expression[Last]("last_value", true), - expression[Max]("max"), - expressionBuilder("max_by", MaxByBuilder), - expression[Average]("mean", true), - expression[Min]("min"), - expressionBuilder("min_by", MinByBuilder), - expression[Percentile]("percentile"), - expressionBuilder("percentile_cont", PercentileContBuilder), - expressionBuilder("percentile_disc", PercentileDiscBuilder), - expression[Median]("median"), - expression[Skewness]("skewness"), - expression[ApproximatePercentile]("percentile_approx"), - expression[ApproximatePercentile]("approx_percentile", true), - expression[HistogramNumeric]("histogram_numeric"), - expression[StddevSamp]("std", true), - expression[StddevSamp]("stddev", true), - expression[StddevPop]("stddev_pop"), - expression[StddevSamp]("stddev_samp"), - expression[Sum]("sum"), - expression[VarianceSamp]("variance", true), - expression[VariancePop]("var_pop"), - expression[VarianceSamp]("var_samp"), - expression[CollectList]("collect_list"), - expression[CollectList]("array_agg", true, Some("3.3.0")), - expression[CollectSet]("collect_set"), - expression[CollectUnion]("collect_union"), - expression[ListAgg]("listagg"), - expression[ListAgg]("string_agg", setAlias = true), - expressionBuilder("count_min_sketch", CountMinSketchAggExpressionBuilder), - expression[BoolAnd]("every", true), - expression[BoolAnd]("bool_and"), - expression[BoolOr]("any", true), - expression[BoolOr]("some", true), - expression[BoolOr]("bool_or"), - expression[RegrCount]("regr_count"), - expression[RegrAvgX]("regr_avgx"), - expression[RegrAvgY]("regr_avgy"), - expression[RegrR2]("regr_r2"), - expression[RegrSXX]("regr_sxx"), - expression[RegrSXY]("regr_sxy"), - expression[RegrSYY]("regr_syy"), - expression[RegrSlope]("regr_slope"), - expression[RegrIntercept]("regr_intercept"), - expressionBuilder("mode", ModeBuilder), - expression[HllSketchAgg]("hll_sketch_agg"), - expression[HllUnionAgg]("hll_union_agg"), - expression[ApproxTopK]("approx_top_k"), - expression[ThetaSketchAgg]("theta_sketch_agg"), - expression[ThetaUnionAgg]("theta_union_agg"), - expression[ThetaIntersectionAgg]("theta_intersection_agg"), - expression[ApproxTopKAccumulate]("approx_top_k_accumulate"), - expression[ApproxTopKCombine]("approx_top_k_combine"), - expression[KllSketchAggBigint]("kll_sketch_agg_bigint"), - expression[KllSketchAggFloat]("kll_sketch_agg_float"), - expression[KllSketchAggDouble]("kll_sketch_agg_double"), - expression[KllMergeAggBigint]("kll_merge_agg_bigint"), - expression[KllMergeAggFloat]("kll_merge_agg_float"), - expression[KllMergeAggDouble]("kll_merge_agg_double"), - expression[TupleIntersectionAggDouble]("tuple_intersection_agg_double"), - expression[TupleIntersectionAggInteger]("tuple_intersection_agg_integer"), - expressionBuilder("tuple_sketch_agg_double", TupleSketchAggDoubleExpressionBuilder), - expressionBuilder("tuple_sketch_agg_integer", TupleSketchAggIntegerExpressionBuilder), - expressionBuilder("tuple_union_agg_double", TupleUnionAggDoubleExpressionBuilder), - expressionBuilder("tuple_union_agg_integer", TupleUnionAggIntegerExpressionBuilder) - ) - - private def vectorExpressions: Seq[FunctionRegistryEntry] = Seq( - // vector functions - expression[VectorCosineSimilarity]("vector_cosine_similarity"), - expression[VectorInnerProduct]("vector_inner_product"), - expression[VectorL2Distance]("vector_l2_distance"), - expression[VectorNorm]("vector_norm"), - expression[VectorNormalize]("vector_normalize"), - expression[VectorAvg]("vector_avg"), - expression[VectorSum]("vector_sum") + expression[Unhex]("unhex") ) private def stringExpressions: Seq[FunctionRegistryEntry] = Seq( // string functions + expression[RandStr]("randstr"), + expression[TryToBinary]("try_to_binary"), expression[Ascii]("ascii"), expression[Chr]("char", true), expression[Chr]("chr"), @@ -639,7 +546,6 @@ object FunctionRegistry { expression[TryToNumber]("try_to_number"), expressionBuilder("to_char", ToCharacterBuilder), expressionBuilder("to_varchar", ToCharacterBuilder, setAlias = true, Some("3.5.0")), - expression[GetJsonObject]("get_json_object"), expression[InitCap]("initcap"), expressionBuilder("instr", StringInstrExpressionBuilder), expression[Lower]("lcase", true), @@ -648,14 +554,11 @@ object FunctionRegistry { expression[Levenshtein]("levenshtein"), expression[JaroWinkler]("jaro_winkler_similarity"), expression[Luhncheck]("luhn_check"), - expression[Like]("like"), - expression[ILike]("ilike"), expression[Lower]("lower"), expression[OctetLength]("octet_length"), expression[StringLocate]("locate"), expressionBuilder("lpad", LPadExpressionBuilder), expression[StringTrimLeft]("ltrim"), - expression[JsonTuple]("json_tuple"), expression[StringLocate]("position", true, Some("2.3.0")), expression[FormatString]("printf", true), expression[RegExpExtract]("regexp_extract"), @@ -664,9 +567,6 @@ object FunctionRegistry { expression[StringRepeat]("repeat"), expression[StringReplace]("replace"), expression[Overlay]("overlay"), - expression[RLike]("rlike"), - expression[RLike]("regexp_like", true, Some("3.2.0")), - expression[RLike]("regexp", true, Some("3.2.0")), expressionBuilder("rpad", RPadExpressionBuilder), expression[StringTrimRight]("rtrim"), expression[Sentences]("sentences"), @@ -685,17 +585,7 @@ object FunctionRegistry { expression[Upper]("ucase", true), expression[UnBase64]("unbase64"), expression[UnBase32]("from_base32"), - expression[Unhex]("unhex"), expression[Upper]("upper"), - expression[XPathList]("xpath"), - expression[XPathBoolean]("xpath_boolean"), - expression[XPathDouble]("xpath_double"), - expression[XPathDouble]("xpath_number", true), - expression[XPathFloat]("xpath_float"), - expression[XPathInt]("xpath_int"), - expression[XPathLong]("xpath_long"), - expression[XPathShort]("xpath_short"), - expression[XPathString]("xpath_string"), expression[RegExpCount]("regexp_count"), expression[RegExpSubStr]("regexp_substr"), expression[RegExpInStr]("regexp_instr"), @@ -704,19 +594,34 @@ object FunctionRegistry { expression[ValidateUTF8]("validate_utf8"), expression[TryValidateUTF8]("try_validate_utf8"), expression[Quote]("quote"), - expression[Normalize]("normalize") + expression[Normalize]("normalize"), + expression[ToBinary]("to_binary"), + expressionBuilder("mask", MaskExpressionBuilder) ) - private def urlExpressions: Seq[FunctionRegistryEntry] = Seq( - // url functions - expression[UrlEncode]("url_encode"), - expression[UrlDecode]("url_decode"), - expression[ParseUrl]("parse_url"), - expression[TryParseUrl]("try_parse_url") + private def bitwiseExpressions: Seq[FunctionRegistryEntry] = Seq( + // bitwise functions + expression[ShiftLeft]("shiftleft"), + expression[ShiftRight]("shiftright"), + expression[ShiftRightUnsigned]("shiftrightunsigned"), + expression[BitwiseAnd]("&"), + expression[BitwiseNot]("~"), + expression[BitwiseOr]("|"), + expression[BitwiseXor]("^"), + expression[ShiftLeft]("<<", true, Some("4.0.0")), + expression[ShiftRight](">>", true, Some("4.0.0")), + expression[ShiftRightUnsigned](">>>", true, Some("4.0.0")), + expression[BitwiseCount]("bit_count"), + expression[BitwiseGet]("bit_get"), + expression[BitwiseGet]("getbit", true) ) private def datetimeExpressions: Seq[FunctionRegistryEntry] = Seq( // datetime functions + expressionBuilder("try_to_timestamp", TryToTimestampExpressionBuilder, setAlias = true), + expressionBuilder("try_to_date", TryToDateExpressionBuilder, setAlias = true), + expressionBuilder("try_to_time", TryToTimeExpressionBuilder, setAlias = true), + expression[TryMakeInterval]("try_make_interval"), expression[AddMonths]("add_months"), expression[CurrentDate]("current_date"), expressionBuilder("curdate", CurDateExpressionBuilder, setAlias = true), @@ -749,7 +654,6 @@ object FunctionRegistry { expression[ParseToDate]("to_date"), expression[TimeDiff]("time_diff"), expression[ToTime]("to_time"), - expression[ToBinary]("to_binary"), expression[ToUnixTimestamp]("to_unix_timestamp"), expression[ToUTCTimestamp]("to_utc_timestamp"), // We keep the 2 expression builders below to have different function docs. @@ -806,8 +710,47 @@ object FunctionRegistry { expressionBuilder("time_bucket", TimeBucketExpressionBuilder) ) + private def hashExpressions: Seq[FunctionRegistryEntry] = Seq( + // hash functions + expression[Crc32]("crc32"), + expression[Md5]("md5"), + expression[Murmur3Hash]("hash"), + expression[XxHash64]("xxhash64"), + expression[Xxh364]("xxh3_64"), + expression[Xxh3128]("xxh3_128"), + expression[Sha1]("sha", true), + expression[Sha1]("sha1"), + expression[Sha2]("sha2") + ) + private def collectionExpressions: Seq[FunctionRegistryEntry] = Seq( // collection functions + expression[TryElementAt]("try_element_at"), + expression[ElementAt]("element_at"), + expression[Size]("size"), + expression[Size]("cardinality", true, Some("2.4.0")), + expression[Reverse]("reverse"), + expression[Concat]("concat") + ) + + private def lambdaExpressions: Seq[FunctionRegistryEntry] = Seq( + // lambda functions + expression[ArraySort]("array_sort"), + expression[ArrayTransform]("transform"), + expression[MapFilter]("map_filter"), + expression[ArrayFilter]("filter"), + expression[ArrayExists]("exists"), + expression[ArrayForAll]("forall"), + expression[ArrayAggregate]("aggregate"), + expression[ArrayAggregate]("reduce", setAlias = true, Some("3.4.0")), + expression[TransformValues]("transform_values"), + expression[TransformKeys]("transform_keys"), + expression[MapZipWith]("map_zip_with"), + expression[ZipWith]("zip_with") + ) + + private def arrayExpressions: Seq[FunctionRegistryEntry] = Seq( + // array functions expression[CreateArray]("array"), expression[ArrayContains]("array_contains"), expression[ArraysOverlap]("arrays_overlap"), @@ -816,68 +759,255 @@ object FunctionRegistry { expression[ArrayJoin]("array_join"), expression[ArrayPosition]("array_position"), expression[ArraySize]("array_size"), - expression[ArraySort]("array_sort"), expression[ArrayExcept]("array_except"), expression[ArrayUnion]("array_union"), expression[ArrayCompact]("array_compact"), - expression[CreateMap]("map"), - expression[CreateNamedStruct]("named_struct"), - expression[ElementAt]("element_at"), - expression[MapContainsKey]("map_contains_key"), - expression[MapFromArrays]("map_from_arrays"), - expression[MapKeys]("map_keys"), - expression[MapValues]("map_values"), - expression[MapEntries]("map_entries"), - expression[MapFromEntries]("map_from_entries"), - expression[MapConcat]("map_concat"), - expression[Size]("size"), expression[Slice]("slice"), expression[TrimArray]("trim_array"), - expression[Size]("cardinality", true, Some("2.4.0")), expression[ArraysZip]("arrays_zip"), expression[SortArray]("sort_array"), expression[Shuffle]("shuffle"), expression[ArrayMin]("array_min"), expression[ArrayMax]("array_max"), expression[ArrayAppend]("array_append"), - expression[Reverse]("reverse"), - expression[Concat]("concat"), expression[Flatten]("flatten"), expression[Sequence]("sequence"), expression[ArrayRepeat]("array_repeat"), expression[ArrayRemove]("array_remove"), expression[ArrayPrepend]("array_prepend"), expression[ArrayDistinct]("array_distinct"), - expression[ArrayTransform]("transform"), - expression[MapFilter]("map_filter"), - expression[ArrayFilter]("filter"), - expression[ArrayExists]("exists"), - expression[ArrayForAll]("forall"), - expression[ArrayAggregate]("aggregate"), - expression[ArrayAggregate]("reduce", setAlias = true, Some("3.4.0")), - expression[TransformValues]("transform_values"), - expression[TransformKeys]("transform_keys"), - expression[MapZipWith]("map_zip_with"), - expression[ZipWith]("zip_with"), - expression[Get]("get"), + expression[Get]("get") + ) + private def structExpressions: Seq[FunctionRegistryEntry] = Seq( + // struct functions + expression[CreateNamedStruct]("named_struct"), CreateStruct.registryEntry ) - private def miscExpressions: Seq[FunctionRegistryEntry] = Seq( - // misc functions - expression[AssertTrue]("assert_true"), - expressionBuilder("raise_error", RaiseErrorExpressionBuilder), - expression[Crc32]("crc32"), - expression[Md5]("md5"), + private def mapExpressions: Seq[FunctionRegistryEntry] = Seq( + // map functions + expression[StringToMap]("str_to_map"), + expression[CreateMap]("map"), + expression[MapContainsKey]("map_contains_key"), + expression[MapFromArrays]("map_from_arrays"), + expression[MapKeys]("map_keys"), + expression[MapValues]("map_values"), + expression[MapEntries]("map_entries"), + expression[MapFromEntries]("map_from_entries"), + expression[MapConcat]("map_concat") + ) + + private def aggregateExpressions: Seq[FunctionRegistryEntry] = Seq( + // aggregate functions + expressionBuilder("try_avg", TryAverageExpressionBuilder, setAlias = true), + expressionBuilder("try_sum", TrySumExpressionBuilder, setAlias = true), + expression[HyperLogLogPlusPlus]("approx_count_distinct"), + expression[Average]("avg"), + expression[Corr]("corr"), + expression[Count]("count"), + expression[CountIf]("count_if"), + expression[CovPopulation]("covar_pop"), + expression[CovSample]("covar_samp"), + expression[First]("first"), + expression[First]("first_value", true), + expression[AnyValue]("any_value"), + expression[Kurtosis]("kurtosis"), + expression[Last]("last"), + expression[Last]("last_value", true), + expression[Max]("max"), + expressionBuilder("max_by", MaxByBuilder), + expression[Average]("mean", true), + expression[Min]("min"), + expressionBuilder("min_by", MinByBuilder), + expression[Percentile]("percentile"), + expressionBuilder("percentile_cont", PercentileContBuilder), + expressionBuilder("percentile_disc", PercentileDiscBuilder), + expression[Median]("median"), + expression[Skewness]("skewness"), + expression[ApproximatePercentile]("percentile_approx"), + expression[ApproximatePercentile]("approx_percentile", true), + expression[HistogramNumeric]("histogram_numeric"), + expression[StddevSamp]("std", true), + expression[StddevSamp]("stddev", true), + expression[StddevPop]("stddev_pop"), + expression[StddevSamp]("stddev_samp"), + expression[Sum]("sum"), + expression[VarianceSamp]("variance", true), + expression[VariancePop]("var_pop"), + expression[VarianceSamp]("var_samp"), + expression[CollectList]("collect_list"), + expression[CollectList]("array_agg", true, Some("3.3.0")), + expression[CollectSet]("collect_set"), + expression[CollectUnion]("collect_union"), + expression[ListAgg]("listagg"), + expression[ListAgg]("string_agg", setAlias = true), + expressionBuilder("count_min_sketch", CountMinSketchAggExpressionBuilder), + expression[BoolAnd]("every", true), + expression[BoolAnd]("bool_and"), + expression[BoolOr]("any", true), + expression[BoolOr]("some", true), + expression[BoolOr]("bool_or"), + expression[RegrCount]("regr_count"), + expression[RegrAvgX]("regr_avgx"), + expression[RegrAvgY]("regr_avgy"), + expression[RegrR2]("regr_r2"), + expression[RegrSXX]("regr_sxx"), + expression[RegrSXY]("regr_sxy"), + expression[RegrSYY]("regr_syy"), + expression[RegrSlope]("regr_slope"), + expression[RegrIntercept]("regr_intercept"), + expressionBuilder("mode", ModeBuilder), + expression[HllSketchAgg]("hll_sketch_agg"), + expression[HllUnionAgg]("hll_union_agg"), + expression[ApproxTopK]("approx_top_k"), + expression[ThetaSketchAgg]("theta_sketch_agg"), + expression[ThetaUnionAgg]("theta_union_agg"), + expression[ThetaIntersectionAgg]("theta_intersection_agg"), + expression[ApproxTopKAccumulate]("approx_top_k_accumulate"), + expression[ApproxTopKCombine]("approx_top_k_combine"), + expression[KllSketchAggBigint]("kll_sketch_agg_bigint"), + expression[KllSketchAggFloat]("kll_sketch_agg_float"), + expression[KllSketchAggDouble]("kll_sketch_agg_double"), + expression[KllMergeAggBigint]("kll_merge_agg_bigint"), + expression[KllMergeAggFloat]("kll_merge_agg_float"), + expression[KllMergeAggDouble]("kll_merge_agg_double"), + expression[TupleIntersectionAggDouble]("tuple_intersection_agg_double"), + expression[TupleIntersectionAggInteger]("tuple_intersection_agg_integer"), + expressionBuilder("tuple_sketch_agg_double", TupleSketchAggDoubleExpressionBuilder), + expressionBuilder("tuple_sketch_agg_integer", TupleSketchAggIntegerExpressionBuilder), + expressionBuilder("tuple_union_agg_double", TupleUnionAggDoubleExpressionBuilder), + expressionBuilder("tuple_union_agg_integer", TupleUnionAggIntegerExpressionBuilder), + expression[Measure]("measure"), + expression[Grouping]("grouping"), + expression[GroupingID]("grouping_id"), + expression[BitAndAgg]("bit_and"), + expression[BitOrAgg]("bit_or"), + expression[BitXorAgg]("bit_xor"), + expression[BitmapConstructAgg]("bitmap_construct_agg"), + expression[BitmapOrAgg]("bitmap_or_agg"), + expression[BitmapAndAgg]("bitmap_and_agg"), + expression[BitmapXorAgg]("bitmap_xor_agg") + ) + + private def windowExpressions: Seq[FunctionRegistryEntry] = Seq( + // window functions + expression[Lead]("lead"), + expression[Lag]("lag"), + expression[RowNumber]("row_number"), + expression[CumeDist]("cume_dist"), + expression[NthValue]("nth_value"), + expression[NTile]("ntile"), + expression[Rank]("rank"), + expression[DenseRank]("dense_rank"), + expression[PercentRank]("percent_rank"), + expressionBuilder("counter_diff", CounterDiffExpressionBuilder) + ) + + private def generatorExpressions: Seq[FunctionRegistryEntry] = Seq( + // generator functions + expressionBuilder("explode", ExplodeExpressionBuilder), + expressionGeneratorBuilderOuter("explode_outer", ExplodeExpressionBuilder), + expressionBuilder("inline", InlineExpressionBuilder), + expressionGeneratorBuilderOuter("inline_outer", InlineExpressionBuilder), + expressionBuilder("posexplode", PosExplodeExpressionBuilder), + expressionGeneratorBuilderOuter("posexplode_outer", PosExplodeExpressionBuilder), + expression[Stack]("stack") + ) + + private def conversionExpressions: Seq[FunctionRegistryEntry] = Seq( + // conversion functions + expression[Cast]("cast"), + // Cast aliases (SPARK-16730) + castAlias("boolean", BooleanType), + castAlias("tinyint", ByteType), + castAlias("smallint", ShortType), + castAlias("int", IntegerType), + castAlias("bigint", LongType), + castAlias("float", FloatType), + castAlias("double", DoubleType), + castAlias("decimal", DecimalType.USER_DEFAULT), + castAlias("date", DateType), + castAlias("timestamp", TimestampType), + castAlias("time", TimeType()), + castAlias("binary", BinaryType), + castAlias("string", StringType) + ) + + private def csvExpressions: Seq[FunctionRegistryEntry] = Seq( + // CSV functions + expression[CsvToStructs]("from_csv"), + expression[SchemaOfCsv]("schema_of_csv"), + expression[StructsToCsv]("to_csv") + ) + + private def jsonExpressions: Seq[FunctionRegistryEntry] = Seq( + // JSON functions + expression[GetJsonObject]("get_json_object"), + expression[JsonTuple]("json_tuple"), + expression[StructsToJson]("to_json"), + expression[JsonToStructs]("from_json"), + expression[SchemaOfJson]("schema_of_json"), + expression[LengthOfJsonArray]("json_array_length"), + expression[JsonObjectKeys]("json_object_keys"), + expression[JsonTypeof]("json_typeof") + ) + + private def variantExpressions: Seq[FunctionRegistryEntry] = Seq( + // variant functions + expressionBuilder("parse_json", ParseJsonExpressionBuilder), + expressionBuilder("try_parse_json", TryParseJsonExpressionBuilder), + expression[IsVariantNull]("is_variant_null"), + expressionBuilder("variant_get", VariantGetExpressionBuilder), + expressionBuilder("try_variant_get", TryVariantGetExpressionBuilder), + expression[SchemaOfVariant]("schema_of_variant"), + expression[SchemaOfVariantAgg]("schema_of_variant_agg"), + expression[ToVariantObject]("to_variant_object"), + expression[VariantFromArrays]("variant_from_arrays"), + expression[VariantFromEntries]("variant_from_entries"), + expression[IsValidVariant]("is_valid_variant"), + expression[VariantDelete]("variant_delete"), + expressionBuilder("variant_insert", VariantInsertExpressionBuilder), + expressionBuilder("try_variant_insert", TryVariantInsertExpressionBuilder), + expressionBuilder("variant_set", VariantSetExpressionBuilder), + expressionBuilder("try_variant_set", TryVariantSetExpressionBuilder), + expressionBuilder("variant_array_append", VariantArrayAppendExpressionBuilder), + expressionBuilder("try_variant_array_append", TryVariantArrayAppendExpressionBuilder), + expressionBuilder("variant_strip_nulls", VariantStripNullsExpressionBuilder) + ) + + private def xmlExpressions: Seq[FunctionRegistryEntry] = Seq( + // XML functions + expression[XPathList]("xpath"), + expression[XPathBoolean]("xpath_boolean"), + expression[XPathDouble]("xpath_double"), + expression[XPathDouble]("xpath_number", true), + expression[XPathFloat]("xpath_float"), + expression[XPathInt]("xpath_int"), + expression[XPathLong]("xpath_long"), + expression[XPathShort]("xpath_short"), + expression[XPathString]("xpath_string"), + expression[XmlToStructs]("from_xml"), + expression[SchemaOfXml]("schema_of_xml"), + expression[StructsToXml]("to_xml") + ) + + private def urlExpressions: Seq[FunctionRegistryEntry] = Seq( + // URL functions + expression[TryUrlDecode]("try_url_decode"), + expression[UrlEncode]("url_encode"), + expression[UrlDecode]("url_decode"), + expression[ParseUrl]("parse_url"), + expression[TryParseUrl]("try_parse_url") + ) + + private def miscExpressions: Seq[FunctionRegistryEntry] = Seq( + // misc functions + expression[TryAesDecrypt]("try_aes_decrypt"), + expression[TryReflect]("try_reflect"), + expression[AssertTrue]("assert_true"), + expressionBuilder("raise_error", RaiseErrorExpressionBuilder), expression[Uuid]("uuid"), - expression[Murmur3Hash]("hash"), - expression[XxHash64]("xxhash64"), - expression[Xxh364]("xxh3_64"), - expression[Xxh3128]("xxh3_128"), - expression[Sha1]("sha", true), - expression[Sha1]("sha1"), - expression[Sha2]("sha2"), expression[AesEncrypt]("aes_encrypt"), expression[AesDecrypt]("aes_decrypt"), expression[Hmac]("hmac"), @@ -897,12 +1027,17 @@ object FunctionRegistry { expression[CallMethodViaReflection]("java_method", true), expression[SparkVersion]("version"), expression[TypeOf]("typeof"), - expression[EqualNull]("equal_null"), - expression[Measure]("measure") + expression[BitmapBucketNumber]("bitmap_bucket_number"), + expression[BitmapBitPosition]("bitmap_bit_position"), + expression[BitmapCount]("bitmap_count"), + expression[BitmapAnd]("bitmap_and"), + expression[BitmapOr]("bitmap_or"), + expression[BitmapAndNot]("bitmap_andnot"), + expression[BitmapXor]("bitmap_xor") ) private def dataSketchExpressions: Seq[FunctionRegistryEntry] = Seq( - // datasketch functions + // Datasketch functions expression[HllSketchEstimate]("hll_sketch_estimate"), expression[HllUnion]("hll_union"), expression[ThetaSketchEstimate]("theta_sketch_estimate"), @@ -945,114 +1080,21 @@ object FunctionRegistry { expression[KllSketchGetRankDouble]("kll_sketch_get_rank_double") ) - private def groupingExpressions: Seq[FunctionRegistryEntry] = Seq( - // grouping sets - expression[Grouping]("grouping"), - expression[GroupingID]("grouping_id") - ) - - private def windowExpressions: Seq[FunctionRegistryEntry] = Seq( - // window functions - expression[Lead]("lead"), - expression[Lag]("lag"), - expression[RowNumber]("row_number"), - expression[CumeDist]("cume_dist"), - expression[NthValue]("nth_value"), - expression[NTile]("ntile"), - expression[Rank]("rank"), - expression[DenseRank]("dense_rank"), - expression[PercentRank]("percent_rank"), - expressionBuilder("counter_diff", CounterDiffExpressionBuilder) - ) - - private def predicateExpressions: Seq[FunctionRegistryEntry] = Seq( - // predicates - expression[Between]("between"), - expression[And]("and"), - expression[In]("in"), - expression[Not]("not"), - expression[Or]("or") - ) - - private def comparisonExpressions: Seq[FunctionRegistryEntry] = Seq( - // comparison operators - expression[EqualNullSafe]("<=>"), - expression[EqualTo]("="), - expression[EqualTo]("=="), - expression[GreaterThan](">"), - expression[GreaterThanOrEqual](">="), - expression[LessThan]("<"), - expression[LessThanOrEqual]("<="), - expression[Not]("!") - ) - - private def bitwiseExpressions: Seq[FunctionRegistryEntry] = Seq( - // bitwise - expression[BitwiseAnd]("&"), - expression[BitwiseNot]("~"), - expression[BitwiseOr]("|"), - expression[BitwiseXor]("^"), - expression[ShiftLeft]("<<", true, Some("4.0.0")), - expression[ShiftRight](">>", true, Some("4.0.0")), - expression[ShiftRightUnsigned](">>>", true, Some("4.0.0")), - expression[BitwiseCount]("bit_count"), - expression[BitAndAgg]("bit_and"), - expression[BitOrAgg]("bit_or"), - expression[BitXorAgg]("bit_xor"), - expression[BitwiseGet]("bit_get"), - expression[BitwiseGet]("getbit", true) - ) - - private def bitmapExpressions: Seq[FunctionRegistryEntry] = Seq( - // bitmap functions and aggregates - expression[BitmapBucketNumber]("bitmap_bucket_number"), - expression[BitmapBitPosition]("bitmap_bit_position"), - expression[BitmapConstructAgg]("bitmap_construct_agg"), - expression[BitmapCount]("bitmap_count"), - expression[BitmapAnd]("bitmap_and"), - expression[BitmapOr]("bitmap_or"), - expression[BitmapAndNot]("bitmap_andnot"), - expression[BitmapXor]("bitmap_xor"), - expression[BitmapOrAgg]("bitmap_or_agg"), - expression[BitmapAndAgg]("bitmap_and_agg"), - expression[BitmapXorAgg]("bitmap_xor_agg") - ) - - private def jsonExpressions: Seq[FunctionRegistryEntry] = Seq( - // json - expression[StructsToJson]("to_json"), - expression[JsonToStructs]("from_json"), - expression[SchemaOfJson]("schema_of_json"), - expression[LengthOfJsonArray]("json_array_length"), - expression[JsonObjectKeys]("json_object_keys"), - expression[JsonTypeof]("json_typeof") + private def avroExpressions: Seq[FunctionRegistryEntry] = Seq( + // Avro functions + expression[FromAvro]("from_avro"), + expression[ToAvro]("to_avro"), + expression[SchemaOfAvro]("schema_of_avro") ) - private def variantExpressions: Seq[FunctionRegistryEntry] = Seq( - // Variant - expressionBuilder("parse_json", ParseJsonExpressionBuilder), - expressionBuilder("try_parse_json", TryParseJsonExpressionBuilder), - expression[IsVariantNull]("is_variant_null"), - expressionBuilder("variant_get", VariantGetExpressionBuilder), - expressionBuilder("try_variant_get", TryVariantGetExpressionBuilder), - expression[SchemaOfVariant]("schema_of_variant"), - expression[SchemaOfVariantAgg]("schema_of_variant_agg"), - expression[ToVariantObject]("to_variant_object"), - expression[VariantFromArrays]("variant_from_arrays"), - expression[VariantFromEntries]("variant_from_entries"), - expression[IsValidVariant]("is_valid_variant"), - expression[VariantDelete]("variant_delete"), - expressionBuilder("variant_insert", VariantInsertExpressionBuilder), - expressionBuilder("try_variant_insert", TryVariantInsertExpressionBuilder), - expressionBuilder("variant_set", VariantSetExpressionBuilder), - expressionBuilder("try_variant_set", TryVariantSetExpressionBuilder), - expressionBuilder("variant_array_append", VariantArrayAppendExpressionBuilder), - expressionBuilder("try_variant_array_append", TryVariantArrayAppendExpressionBuilder), - expressionBuilder("variant_strip_nulls", VariantStripNullsExpressionBuilder) + private def protobufExpressions: Seq[FunctionRegistryEntry] = Seq( + // Protobuf functions + expression[FromProtobuf]("from_protobuf"), + expression[ToProtobuf]("to_protobuf") ) private def spatialExpressions: Seq[FunctionRegistryEntry] = Seq( - // Spatial + // ST geospatial functions expression[ST_AsBinary]("st_asbinary"), expression[ST_GeogFromWKB]("st_geogfromwkb"), expression[ST_GeomFromWKB]("st_geomfromwkb"), @@ -1060,86 +1102,49 @@ object FunctionRegistry { expression[ST_SetSrid]("st_setsrid") ) - private def castExpressions: Seq[FunctionRegistryEntry] = Seq( - // cast - expression[Cast]("cast"), - // Cast aliases (SPARK-16730) - castAlias("boolean", BooleanType), - castAlias("tinyint", ByteType), - castAlias("smallint", ShortType), - castAlias("int", IntegerType), - castAlias("bigint", LongType), - castAlias("float", FloatType), - castAlias("double", DoubleType), - castAlias("decimal", DecimalType.USER_DEFAULT), - castAlias("date", DateType), - castAlias("timestamp", TimestampType), - castAlias("time", TimeType()), - castAlias("binary", BinaryType), - castAlias("string", StringType) - ) - - private def maskExpressions: Seq[FunctionRegistryEntry] = Seq( - // mask functions - expressionBuilder("mask", MaskExpressionBuilder) - ) - - private def csvExpressions: Seq[FunctionRegistryEntry] = Seq( - // csv - expression[CsvToStructs]("from_csv"), - expression[SchemaOfCsv]("schema_of_csv"), - expression[StructsToCsv]("to_csv") - ) - - private def xmlExpressions: Seq[FunctionRegistryEntry] = Seq( - // Xml - expression[XmlToStructs]("from_xml"), - expression[SchemaOfXml]("schema_of_xml"), - expression[StructsToXml]("to_xml") - ) - - private def avroExpressions: Seq[FunctionRegistryEntry] = Seq( - // Avro - expression[FromAvro]("from_avro"), - expression[ToAvro]("to_avro"), - expression[SchemaOfAvro]("schema_of_avro") - ) - - private def protobufExpressions: Seq[FunctionRegistryEntry] = Seq( - // Protobuf - expression[FromProtobuf]("from_protobuf"), - expression[ToProtobuf]("to_protobuf") + private def vectorExpressions: Seq[FunctionRegistryEntry] = Seq( + // vector functions + expression[VectorCosineSimilarity]("vector_cosine_similarity"), + expression[VectorInnerProduct]("vector_inner_product"), + expression[VectorL2Distance]("vector_l2_distance"), + expression[VectorNorm]("vector_norm"), + expression[VectorNormalize]("vector_normalize"), + expression[VectorAvg]("vector_avg"), + expression[VectorSum]("vector_sum") ) - // Keep expression groups in separate methods to limit the bytecode size of the static - // initializer and leave enough headroom for coverage instrumentation. + // Keep registry entries aligned with their ExpressionInfo groups. Public APIs and documentation + // use the corresponding taxonomy where applicable. Separate methods also limit the bytecode size + // of the static initializer and leave enough headroom for coverage instrumentation. val expressions: Map[String, (ExpressionInfo, FunctionBuilder)] = Seq( - miscNonAggregateExpressions, + conditionalExpressions, + predicateExpressions, mathExpressions, - tryExpressions, - aggregateExpressions, - vectorExpressions, stringExpressions, - urlExpressions, + bitwiseExpressions, datetimeExpressions, + hashExpressions, collectionExpressions, - miscExpressions, - dataSketchExpressions, - groupingExpressions, + lambdaExpressions, + arrayExpressions, + structExpressions, + mapExpressions, + aggregateExpressions, windowExpressions, - predicateExpressions, - comparisonExpressions, - bitwiseExpressions, - bitmapExpressions, + generatorExpressions, + conversionExpressions, + csvExpressions, jsonExpressions, variantExpressions, - spatialExpressions, - castExpressions, - maskExpressions, - csvExpressions, xmlExpressions, + urlExpressions, + miscExpressions, + dataSketchExpressions, avroExpressions, - protobufExpressions).flatten.toMap + protobufExpressions, + spatialExpressions, + vectorExpressions + ).flatten.toMap // BuiltinRegistryMixin normalizes any name to the builtin 3-part key (system.builtin.name). val builtin: SimpleFunctionRegistry = { From 5df40d21a2c74d4cf9032a0920c5b7e6abe46dca Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 2 Sep 2026 12:07:23 +0000 Subject: [PATCH 2/4] [SPARK-59170][SQL] Align Scala and Python function sections --- python/pyspark/sql/functions/builtin.py | 38999 ++++++++-------- .../org/apache/spark/sql/functions.scala | 24211 +++++----- 2 files changed, 31648 insertions(+), 31562 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 54a55f91cd9fb..ae1632d18da2d 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -109,11 +109,6 @@ # even though there might be few exceptions for legacy or inevitable reasons. # If you are fixing other language APIs together, also please note that Scala side is not the case # since it requires making every single overridden definition. -# Public function groups are defined by pyspark.sql.functions.__all__ and mirrored in the API -# reference. -# Section headings in this implementation file are only navigation aids. - - def _get_jvm_function(name: str, sc: "SparkContext") -> Callable: """ Retrieves JVM function identified by name from @@ -176,6 +171,8 @@ def _options_to_str(options: Optional[Mapping[str, Any]] = None) -> Mapping[str, return {key: _to_str(value) for (key, value) in options.items()} return {} +# ---------------------- Normal Functions ---------------------- + @_try_remote_functions def lit(col: Any) -> Column: @@ -357,693 +354,535 @@ def col(col: str) -> Column: @_try_remote_functions -def asc(col: "ColumnOrName") -> Column: +def broadcast(df: "DataFrame") -> "DataFrame": """ - Returns a sort expression for the target column in ascending order. - This function is used in `sort` and `orderBy` functions. + Marks a DataFrame as small enough for use in broadcast joins. - .. versionadded:: 1.3.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - Target column to sort by in the ascending order. - Returns ------- - :class:`~pyspark.sql.Column` - The column specifying the sort order. - - See Also - -------- - :meth:`pyspark.sql.functions.asc_nulls_first` - :meth:`pyspark.sql.functions.asc_nulls_last` + :class:`~pyspark.sql.DataFrame` + DataFrame marked as ready for broadcast join. Examples -------- - Example 1: Sort DataFrame by 'id' column in ascending order. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(sf.asc("id")).show() - +---+-----+ - | id|value| - +---+-----+ - | 2| C| - | 3| A| - | 4| B| - +---+-----+ - - Example 2: Use `asc` in `orderBy` function to sort the DataFrame. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.orderBy(sf.asc("value")).show() - +---+-----+ - | id|value| - +---+-----+ - | 3| A| - | 4| B| - | 2| C| - +---+-----+ - - Example 3: Combine `asc` with `desc` to sort by multiple columns. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], - ... ['id', 'group', 'value']) - >>> df.sort(sf.asc("group"), sf.desc("value")).show() - +---+-----+-----+ - | id|group|value| - +---+-----+-----+ - | 2| A| 4| - | 3| A| 2| - | 1| B| 3| - +---+-----+-----+ + >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") + >>> df_small = spark.range(3) + >>> df_b = sf.broadcast(df_small) + >>> df.join(df_b, df.value == df_small.id).show() + +-----+---+ + |value| id| + +-----+---+ + | 1| 1| + | 2| 2| + +-----+---+ + """ + from py4j.java_gateway import JVMView - Example 4: Implement `asc` from column expression. + from pyspark.sql.dataframe import DataFrame - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(df.id.asc()).show() - +---+-----+ - | id|value| - +---+-----+ - | 2| C| - | 3| A| - | 4| B| - +---+-----+ - """ - return col.asc() if isinstance(col, Column) else _invoke_function("asc", col) + sc = _get_active_spark_context() + return DataFrame(cast(JVMView, sc._jvm).functions.broadcast(df._jdf), df.sparkSession) @_try_remote_functions -def desc(col: "ColumnOrName") -> Column: - """ - Returns a sort expression for the target column in descending order. - This function is used in `sort` and `orderBy` functions. +def expr(str: str) -> Column: + """Parses the expression string into the column that it represents - .. versionadded:: 1.3.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Target column to sort by in the descending order. + str : expression string + expression defined in string. Returns ------- :class:`~pyspark.sql.Column` - The column specifying the sort order. - - See Also - -------- - :meth:`pyspark.sql.functions.desc_nulls_first` - :meth:`pyspark.sql.functions.desc_nulls_last` + column representing the expression. Examples -------- - Example 1: Sort DataFrame by 'id' column in descending order. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(sf.desc("id")).show() - +---+-----+ - | id|value| - +---+-----+ - | 4| B| - | 3| A| - | 2| C| - +---+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([["Alice"], ["Bob"]], ["name"]) + >>> df.select("*", sf.expr("length(name)")).show() + +-----+------------+ + | name|length(name)| + +-----+------------+ + |Alice| 5| + | Bob| 3| + +-----+------------+ + """ + return _invoke_function("expr", str) - Example 2: Use `desc` in `orderBy` function to sort the DataFrame. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.orderBy(sf.desc("value")).show() - +---+-----+ - | id|value| - +---+-----+ - | 2| C| - | 4| B| - | 3| A| - +---+-----+ +@_try_remote_functions +def call_function(funcName: str, *cols: "ColumnOrName") -> Column: + """ + Call a SQL function. - Example 3: Combine `asc` with `desc` to sort by multiple columns. + .. versionadded:: 3.5.0 - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], - ... ['id', 'group', 'value']) - >>> df.sort(sf.desc("group"), sf.asc("value")).show() - +---+-----+-----+ - | id|group|value| - +---+-----+-----+ - | 1| B| 3| - | 3| A| 2| - | 2| A| 4| - +---+-----+-----+ + Parameters + ---------- + funcName : str + function name that follows the SQL identifier syntax (can be quoted, can be qualified) + cols : :class:`~pyspark.sql.Column` or str + column names or :class:`~pyspark.sql.Column`\\s to be used in the function - Example 4: Implement `desc` from column expression. + Returns + ------- + :class:`~pyspark.sql.Column` + result of executed function. - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(df.id.desc()).show() - +---+-----+ - | id|value| - +---+-----+ - | 4| B| - | 3| A| - | 2| C| - +---+-----+ + Examples + -------- + >>> from pyspark.sql.functions import call_udf, col + >>> from pyspark.sql.types import IntegerType, StringType + >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) + >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) + >>> df.select(call_function("intX2", "id")).show() + +---------+ + |intX2(id)| + +---------+ + | 2| + | 4| + | 6| + +---------+ + >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) + >>> df.select(call_function("strX2", col("name"))).show() + +-----------+ + |strX2(name)| + +-----------+ + | aa| + | bb| + | cc| + +-----------+ + >>> df.select(call_function("avg", col("id"))).show() + +-------+ + |avg(id)| + +-------+ + | 2.0| + +-------+ + >>> _ = spark.sql("CREATE FUNCTION custom_avg AS 'test.org.apache.spark.sql.MyDoubleAvg'") + ... # doctest: +SKIP + >>> df.select(call_function("custom_avg", col("id"))).show() + ... # doctest: +SKIP + +------------------------------------+ + |spark_catalog.default.custom_avg(id)| + +------------------------------------+ + | 102.0| + +------------------------------------+ + >>> df.select(call_function("spark_catalog.default.custom_avg", col("id"))).show() + ... # doctest: +SKIP + +------------------------------------+ + |spark_catalog.default.custom_avg(id)| + +------------------------------------+ + | 102.0| + +------------------------------------+ """ - return col.desc() if isinstance(col, Column) else _invoke_function("desc", col) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function("call_function", funcName, _to_seq(sc, cols, _to_java_column)) + + +# ---------------------- Conditional Functions ---------------------- @_try_remote_functions -def sqrt(col: "ColumnOrName") -> Column: - """ - Computes the square root of the specified float value. +def coalesce(*cols: "ColumnOrName") -> Column: + """Returns the first column that is not null. - .. versionadded:: 1.3.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a double. + cols : :class:`~pyspark.sql.Column` or column name + list of columns to work on. + Each a column of any type. Returns ------- :class:`~pyspark.sql.Column` - column for computed results. - Returns a column that evaluates to a double. + value of the first column that is not null. + Returns a column of the same type as the input. Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (-1), (0), (1), (4), (NULL) AS TAB(value)" - ... ).select("*", sf.sqrt("value")).show() - +-----+-----------+ - |value|SQRT(value)| - +-----+-----------+ - | -1| NaN| - | 0| 0.0| - | 1| 1.0| - | 4| 2.0| - | NULL| NULL| - +-----+-----------+ + >>> df = spark.createDataFrame([(None, None), (1, None), (None, 2)], ("a", "b")) + >>> df.show() + +----+----+ + | a| b| + +----+----+ + |NULL|NULL| + | 1|NULL| + |NULL| 2| + +----+----+ + + >>> df.select('*', sf.coalesce("a", df["b"])).show() + +----+----+--------------+ + | a| b|coalesce(a, b)| + +----+----+--------------+ + |NULL|NULL| NULL| + | 1|NULL| 1| + |NULL| 2| 2| + +----+----+--------------+ + + >>> df.select('*', sf.coalesce(df["a"], lit(0.0))).show() + +----+----+----------------+ + | a| b|coalesce(a, 0.0)| + +----+----+----------------+ + |NULL|NULL| 0.0| + | 1|NULL| 1.0| + |NULL| 2| 0.0| + +----+----+----------------+ """ - return _invoke_function_over_columns("sqrt", col) + return _invoke_function_over_seq_of_columns("coalesce", cols) @_try_remote_functions -def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """ - Returns the sum of `left`and `right` and the result is null on overflow. - The acceptable input types are the same with the `+` operator. +def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns col1 if it is not NaN, or col2 if col1 is NaN. - .. versionadded:: 3.5.0 + Both inputs should be floating point columns (:class:`DoubleType` or :class:`FloatType`). + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. - right : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. + col1 : :class:`~pyspark.sql.Column` or column name + first column to check. + A column that evaluates to a double or float. + col2 : :class:`~pyspark.sql.Column` or column name + second column to return if first is NaN. + A column that evaluates to a double or float. + + Returns + ------- + :class:`~pyspark.sql.Column` + value from first column or second if first is NaN . + Returns a column of the same type as the first input. Examples -------- - Example 1: Integer plus Integer. - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(1982, 15), (1990, 2)], ["birth", "age"] - ... ).select("*", sf.try_add("birth", "age")).show() - +-----+---+-------------------+ - |birth|age|try_add(birth, age)| - +-----+---+-------------------+ - | 1982| 15| 1997| - | 1990| 2| 1992| - +-----+---+-------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) + >>> df.select("*", sf.nanvl("a", "b"), sf.nanvl(df.a, df.b)).show() + +---+---+-----------+-----------+ + | a| b|nanvl(a, b)|nanvl(a, b)| + +---+---+-----------+-----------+ + |1.0|NaN| 1.0| 1.0| + |NaN|2.0| 2.0| 2.0| + +---+---+-----------+-----------+ + """ + return _invoke_function_over_columns("nanvl", col1, col2) - Example 2: Date plus Integer. - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-09-30')) AS TAB(date)" - ... ).select("*", sf.try_add("date", sf.lit(1))).show() - +----------+----------------+ - | date|try_add(date, 1)| - +----------+----------------+ - |2015-09-30| 2015-10-01| - +----------+----------------+ +@_try_remote_functions +def when(condition: Column, value: Any) -> Column: + """Evaluates a list of conditions and returns one of multiple possible result expressions. + If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched + conditions. - Example 3: Date plus Interval. + .. versionadded:: 1.4.0 - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" - ... ).select("*", sf.try_add("date", "itvl")).show() - +----------+-----------------+-------------------+ - | date| itvl|try_add(date, itvl)| - +----------+-----------------+-------------------+ - |2015-09-30|INTERVAL '1' YEAR| 2016-09-30| - +----------+-----------------+-------------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Example 4: Interval plus Interval. + Parameters + ---------- + condition : :class:`~pyspark.sql.Column` + a boolean :class:`~pyspark.sql.Column` expression. + A column that evaluates to a boolean. + value : + a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of any type. - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" - ... ).select("*", sf.try_add("itvl1", "itvl2")).show() - +-----------------+-----------------+---------------------+ - | itvl1| itvl2|try_add(itvl1, itvl2)| - +-----------------+-----------------+---------------------+ - |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '3' YEAR| - +-----------------+-----------------+---------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + column representing when expression. + Returns a column of the same type as the input. - Example 5: Overflow results in NULL when ANSI mode is on + See Also + -------- + :meth:`pyspark.sql.Column.when` + :meth:`pyspark.sql.Column.otherwise` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_add(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-------------------------------------------------+ - |try_add(9223372036854775807, 9223372036854775807)| - +-------------------------------------------------+ - | NULL| - +-------------------------------------------------+ + >>> df = spark.range(3) + >>> df.select("*", sf.when(df['id'] == 2, 3).otherwise(4)).show() + +---+------------------------------------+ + | id|CASE WHEN (id = 2) THEN 3 ELSE 4 END| + +---+------------------------------------+ + | 0| 4| + | 1| 4| + | 2| 3| + +---+------------------------------------+ + + >>> df.select("*", sf.when(df.id == 2, df.id + 1)).show() + +---+------------------------------------+ + | id|CASE WHEN (id = 2) THEN (id + 1) END| + +---+------------------------------------+ + | 0| NULL| + | 1| NULL| + | 2| 3| + +---+------------------------------------+ """ - return _invoke_function_over_columns("try_add", left, right) + # Explicitly not using ColumnOrName type here to make reading condition less opaque + if not isinstance(condition, Column): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column", + "arg_name": "condition", + "arg_type": type(condition).__name__, + }, + ) + value = _enum_to_value(value) + v = value._jc if isinstance(value, Column) else _enum_to_value(value) + + return _invoke_function("when", condition._jc, v) @_try_remote_functions -def try_avg(col: "ColumnOrName") -> Column: +def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns the mean calculated from values of a group and the result is null on overflow. + Returns `col2` if `col1` is null, or `col1` otherwise. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric or interval. + col1 : :class:`~pyspark.sql.Column` or str + col2 : :class:`~pyspark.sql.Column` or str Examples -------- - Example 1: Calculating the average age - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) - >>> df.select(sf.try_avg("age")).show() - +------------+ - |try_avg(age)| - +------------+ - | 8.5| - +------------+ - - Example 2: Calculating the average age with None - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.try_avg("age")).show() + >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) + >>> df.select(sf.ifnull(df.e, sf.lit(8))).show() +------------+ - |try_avg(age)| + |ifnull(e, 8)| +------------+ - | 3.0| + | 8| + | 1| +------------+ - - Example 3: Overflow results in NULL when ANSI mode is on - - >>> from decimal import Decimal - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.createDataFrame( - ... [(Decimal("1" * 38),), (Decimal(0),)], "number DECIMAL(38, 0)") - ... df.select(sf.try_avg(df.number)).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +---------------+ - |try_avg(number)| - +---------------+ - | NULL| - +---------------+ """ - return _invoke_function_over_columns("try_avg", col) + return _invoke_function_over_columns("ifnull", col1, col2) @_try_remote_functions -def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns `dividend`/`divisor`. It always performs floating point division. Its result is - always null if `divisor` is 0. + Returns null if `col1` equals to `col2`, or `col1` otherwise. .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - dividend. - A column that evaluates to a numeric or interval. - right : :class:`~pyspark.sql.Column` or column name - divisor. - A column that evaluates to a numeric. + col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. + col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. Examples -------- - Example 1: Integer divided by Integer. - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(6000, 15), (1990, 2), (1234, 0)], ["a", "b"] - ... ).select("*", sf.try_divide("a", "b")).show() - +----+---+----------------+ - | a| b|try_divide(a, b)| - +----+---+----------------+ - |6000| 15| 400.0| - |1990| 2| 995.0| - |1234| 0| NULL| - +----+---+----------------+ - - Example 2: Interval divided by Integer. - - >>> import pyspark.sql.functions as sf - >>> df = spark.range(4).select(sf.make_interval(sf.lit(1)).alias("itvl"), "id") - >>> df.select("*", sf.try_divide("itvl", "id")).show() - +-------+---+--------------------+ - | itvl| id|try_divide(itvl, id)| - +-------+---+--------------------+ - |1 years| 0| NULL| - |1 years| 1| 1 years| - |1 years| 2| 6 months| - |1 years| 3| 4 months| - +-------+---+--------------------+ - - Example 3: Exception during division, resulting in NULL when ANSI mode is on + >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) + >>> df.select('*', sf.nullif(df.a, df.b)).show() + +----+----+------------+ + | a| b|nullif(a, b)| + +----+----+------------+ + |NULL|NULL| NULL| + | 1| 9| 1| + +----+----+------------+ - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_divide("id", sf.lit(0))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-----------------+ - |try_divide(id, 0)| - +-----------------+ - | NULL| - +-----------------+ + >>> df.select('*', sf.nullif('a', 'b')).show() + +----+----+------------+ + | a| b|nullif(a, b)| + +----+----+------------+ + |NULL|NULL| NULL| + | 1| 9| 1| + +----+----+------------+ """ - return _invoke_function_over_columns("try_divide", left, right) + return _invoke_function_over_columns("nullif", col1, col2) @_try_remote_functions -def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def nullifzero(col: "ColumnOrName") -> Column: """ - Returns the remainder after `dividend`/`divisor`. Its result is - always null if `divisor` is 0. + Returns null if `col` is equal to zero, or `col` otherwise. .. versionadded:: 4.0.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - dividend. - A column that evaluates to a numeric. - right : :class:`~pyspark.sql.Column` or column name - divisor. + col : :class:`~pyspark.sql.Column` or column name A column that evaluates to a numeric. Examples -------- - Example 1: Integer divided by Integer. - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(6000, 15), (3, 2), (1234, 0)], ["a", "b"] - ... ).select("*", sf.try_mod("a", "b")).show() - +----+---+-------------+ - | a| b|try_mod(a, b)| - +----+---+-------------+ - |6000| 15| 0| - | 3| 2| 1| - |1234| 0| NULL| - +----+---+-------------+ - - Example 2: Exception during division, resulting in NULL when ANSI mode is on + >>> df = spark.createDataFrame([(0,), (1,)], ["a"]) + >>> df.select('*', sf.nullifzero(df.a)).show() + +---+-------------+ + | a|nullifzero(a)| + +---+-------------+ + | 0| NULL| + | 1| 1| + +---+-------------+ - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_mod("id", sf.lit(0))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +--------------+ - |try_mod(id, 0)| - +--------------+ - | NULL| - +--------------+ + >>> df.select('*', sf.nullifzero('a')).show() + +---+-------------+ + | a|nullifzero(a)| + +---+-------------+ + | 0| NULL| + | 1| 1| + +---+-------------+ """ - return _invoke_function_over_columns("try_mod", left, right) + return _invoke_function_over_columns("nullifzero", col) @_try_remote_functions -def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def nvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns `left`*`right` and the result is null on overflow. The acceptable input types are the - same with the `*` operator. + Returns `col2` if `col1` is null, or `col1` otherwise. .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - multiplicand. - A column that evaluates to a numeric or interval. - right : :class:`~pyspark.sql.Column` or column name - multiplier. - A column that evaluates to a numeric or interval. + col1 : :class:`~pyspark.sql.Column` or column name + col2 : :class:`~pyspark.sql.Column` or column name - Examples + See Also -------- - Example 1: Integer multiplied by Integer. - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(6000, 15), (1990, 2)], ["a", "b"] - ... ).select("*", sf.try_multiply("a", "b")).show() - +----+---+------------------+ - | a| b|try_multiply(a, b)| - +----+---+------------------+ - |6000| 15| 90000| - |1990| 2| 3980| - +----+---+------------------+ - - Example 2: Interval multiplied by Integer. + :meth:`pyspark.sql.functions.nvl2` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.range(6).select(sf.make_interval(sf.col("id"), sf.lit(3)).alias("itvl"), "id") - >>> df.select("*", sf.try_multiply("itvl", "id")).show() - +----------------+---+----------------------+ - | itvl| id|try_multiply(itvl, id)| - +----------------+---+----------------------+ - | 3 months| 0| 0 seconds| - |1 years 3 months| 1| 1 years 3 months| - |2 years 3 months| 2| 4 years 6 months| - |3 years 3 months| 3| 9 years 9 months| - |4 years 3 months| 4| 17 years| - |5 years 3 months| 5| 26 years 3 months| - +----------------+---+----------------------+ - - Example 3: Overflow results in NULL when ANSI mode is on + >>> df = spark.createDataFrame([(None, 8,), (1, 9,)], ["a", "b"]) + >>> df.select('*', sf.nvl(df.a, df.b)).show() + +----+---+---------+ + | a| b|nvl(a, b)| + +----+---+---------+ + |NULL| 8| 8| + | 1| 9| 1| + +----+---+---------+ - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_multiply(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +------------------------------------------------------+ - |try_multiply(9223372036854775807, 9223372036854775807)| - +------------------------------------------------------+ - | NULL| - +------------------------------------------------------+ + >>> df.select('*', sf.nvl('a', 'b')).show() + +----+---+---------+ + | a| b|nvl(a, b)| + +----+---+---------+ + |NULL| 8| 8| + | 1| 9| 1| + +----+---+---------+ """ - return _invoke_function_over_columns("try_multiply", left, right) + return _invoke_function_over_columns("nvl", col1, col2) @_try_remote_functions -def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def nvl2(col1: "ColumnOrName", col2: "ColumnOrName", col3: "ColumnOrName") -> Column: """ - Returns `left`-`right` and the result is null on overflow. The acceptable input types are the - same with the `-` operator. + Returns `col2` if `col1` is not null, or `col3` otherwise. .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. - right : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. + col1 : :class:`~pyspark.sql.Column` or column name + col2 : :class:`~pyspark.sql.Column` or column name + col3 : :class:`~pyspark.sql.Column` or column name - Examples + See Also -------- - Example 1: Integer minus Integer. - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(1982, 15), (1990, 2)], ["birth", "age"] - ... ).select("*", sf.try_subtract("birth", "age")).show() - +-----+---+------------------------+ - |birth|age|try_subtract(birth, age)| - +-----+---+------------------------+ - | 1982| 15| 1967| - | 1990| 2| 1988| - +-----+---+------------------------+ - - Example 2: Date minus Integer. - - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-10-01')) AS TAB(date)" - ... ).select("*", sf.try_subtract("date", sf.lit(1))).show() - +----------+---------------------+ - | date|try_subtract(date, 1)| - +----------+---------------------+ - |2015-10-01| 2015-09-30| - +----------+---------------------+ - - Example 3: Date minus Interval. - - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" - ... ).select("*", sf.try_subtract("date", "itvl")).show() - +----------+-----------------+------------------------+ - | date| itvl|try_subtract(date, itvl)| - +----------+-----------------+------------------------+ - |2015-09-30|INTERVAL '1' YEAR| 2014-09-30| - +----------+-----------------+------------------------+ - - Example 4: Interval minus Interval. + :meth:`pyspark.sql.functions.nvl` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" - ... ).select("*", sf.try_subtract("itvl1", "itvl2")).show() - +-----------------+-----------------+--------------------------+ - | itvl1| itvl2|try_subtract(itvl1, itvl2)| - +-----------------+-----------------+--------------------------+ - |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '-1' YEAR| - +-----------------+-----------------+--------------------------+ - - Example 5: Overflow results in NULL when ANSI mode is on + >>> df = spark.createDataFrame([(None, 8, 6,), (1, 9, 9,)], ["a", "b", "c"]) + >>> df.select('*', sf.nvl2(df.a, df.b, df.c)).show() + +----+---+---+-------------+ + | a| b| c|nvl2(a, b, c)| + +----+---+---+-------------+ + |NULL| 8| 6| 6| + | 1| 9| 9| 9| + +----+---+---+-------------+ - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_subtract(sf.lit(-sys.maxsize), sf.lit(sys.maxsize))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-------------------------------------------------------+ - |try_subtract(-9223372036854775807, 9223372036854775807)| - +-------------------------------------------------------+ - | NULL| - +-------------------------------------------------------+ + >>> df.select('*', sf.nvl2('a', 'b', 'c')).show() + +----+---+---+-------------+ + | a| b| c|nvl2(a, b, c)| + +----+---+---+-------------+ + |NULL| 8| 6| 6| + | 1| 9| 9| 9| + +----+---+---+-------------+ """ - return _invoke_function_over_columns("try_subtract", left, right) + return _invoke_function_over_columns("nvl2", col1, col2, col3) @_try_remote_functions -def try_sum(col: "ColumnOrName") -> Column: +def zeroifnull(col: "ColumnOrName") -> Column: """ - Returns the sum calculated from values of a group and the result is null on overflow. + Returns zero if `col` is null, or `col` otherwise. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric or interval. Examples -------- - Example 1: Calculating the sum of values in a column - - >>> from pyspark.sql import functions as sf - >>> spark.range(10).select(sf.try_sum("id")).show() - +-----------+ - |try_sum(id)| - +-----------+ - | 45| - +-----------+ - - Example 2: Using a plus expression together to calculate the sum - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) - >>> df.select(sf.try_sum(sf.col("A") + sf.col("B"))).show() - +----------------+ - |try_sum((A + B))| - +----------------+ - | 10| - +----------------+ - - Example 3: Calculating the summation of ages with None - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.try_sum("age")).show() - +------------+ - |try_sum(age)| - +------------+ - | 6| - +------------+ - - Example 4: Overflow results in NULL when ANSI mode is on + >>> df = spark.createDataFrame([(None,), (1,)], ["a"]) + >>> df.select('*', sf.zeroifnull(df.a)).show() + +----+-------------+ + | a|zeroifnull(a)| + +----+-------------+ + |NULL| 0| + | 1| 1| + +----+-------------+ - >>> from decimal import Decimal - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.createDataFrame([(Decimal("1" * 38),)] * 10, "number DECIMAL(38, 0)") - ... df.select(sf.try_sum(df.number)).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +---------------+ - |try_sum(number)| - +---------------+ - | NULL| - +---------------+ + >>> df.select('*', sf.zeroifnull('a')).show() + +----+-------------+ + | a|zeroifnull(a)| + +----+-------------+ + |NULL| 0| + | 1| 1| + +----+-------------+ """ - return _invoke_function_over_columns("try_sum", col) + return _invoke_function_over_columns("zeroifnull", col) + + +# ---------------------- Predicate Functions ---------------------- @_try_remote_functions -def abs(col: "ColumnOrName") -> Column: - """ - Mathematical Function: Computes the absolute value of the given column or expression. +def isnan(col: "ColumnOrName") -> Column: + """An expression that returns true if the column is NaN. - .. versionadded:: 1.3.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1051,556 +890,554 @@ def abs(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The target column or expression to compute the absolute value on. - A column that evaluates to a numeric or interval. + target column to compute on. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` - A new column object representing the absolute value of the input. - Returns a column of the same type as the input. + True if value is NaN and False otherwise. + Returns a column that evaluates to a boolean. - Examples + See Also -------- - Example 1: Compute the absolute value of a long column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(-1,), (-2,), (-3,), (None,)], ["value"]) - >>> df.select("*", sf.abs(df.value)).show() - +-----+----------+ - |value|abs(value)| - +-----+----------+ - | -1| 1| - | -2| 2| - | -3| 3| - | NULL| NULL| - +-----+----------+ - - Example 2: Compute the absolute value of a double column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(-1.5,), (-2.5,), (None,), (float("nan"),)], ["value"]) - >>> df.select("*", sf.abs(df.value)).show() - +-----+----------+ - |value|abs(value)| - +-----+----------+ - | -1.5| 1.5| - | -2.5| 2.5| - | NULL| NULL| - | NaN| NaN| - +-----+----------+ - - Example 3: Compute the absolute value of an expression + :meth:`pyspark.sql.functions.isnull` + :meth:`pyspark.sql.functions.isnotnull` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1), (2, -2), (3, 3)], ["id", "value"]) - >>> df.select("*", sf.abs(df.id - df.value)).show() - +---+-----+-----------------+ - | id|value|abs((id - value))| - +---+-----+-----------------+ - | 1| 1| 0| - | 2| -2| 4| - | 3| 3| 0| - +---+-----+-----------------+ + >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) + >>> df.select("*", sf.isnan("a"), sf.isnan(df.b)).show() + +---+---+--------+--------+ + | a| b|isnan(a)|isnan(b)| + +---+---+--------+--------+ + |1.0|NaN| false| true| + |NaN|2.0| true| false| + +---+---+--------+--------+ """ - return _invoke_function_over_columns("abs", col) + return _invoke_function_over_columns("isnan", col) @_try_remote_functions -def mode(col: "ColumnOrName", deterministic: bool = False) -> Column: - """ - Returns the most frequent value in a group. +def isnull(col: "ColumnOrName") -> Column: + """An expression that returns true if the column is null. - .. versionadded:: 3.4.0 + .. versionadded:: 1.6.0 - .. versionchanged:: 4.0.0 - Supports deterministic argument. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. A column of any type. - deterministic : bool, optional - if there are multiple equally-frequent results then return the lowest (defaults to false). - A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - the most frequent value in a group. + True if value is null and False otherwise. + Returns a column that evaluates to a boolean. - Notes - ----- - Supports Spark Connect. + See Also + -------- + :meth:`pyspark.sql.functions.isnan` + :meth:`pyspark.sql.functions.isnotnull` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.mode("year")).sort("course").show() - +------+----------+ - |course|mode(year)| - +------+----------+ - | Java| 2012| - |dotNET| 2012| - +------+----------+ - - When multiple values have the same greatest frequency then either any of values is returned if - deterministic is false or is not defined, or the lowest value is returned if deterministic is - true. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(-10,), (0,), (10,)], ["col"]) - >>> df.select(sf.mode("col", False)).show() # doctest: +SKIP - +---------+ - |mode(col)| - +---------+ - | 0| - +---------+ - - >>> df.select(sf.mode("col", True)).show() - +---------------------------------------+ - |mode() WITHIN GROUP (ORDER BY col DESC)| - +---------------------------------------+ - | -10| - +---------------------------------------+ + >>> df = spark.createDataFrame([(1, None), (None, 2)], ("a", "b")) + >>> df.select("*", sf.isnull("a"), isnull(df.b)).show() + +----+----+-----------+-----------+ + | a| b|(a IS NULL)|(b IS NULL)| + +----+----+-----------+-----------+ + | 1|NULL| false| true| + |NULL| 2| true| false| + +----+----+-----------+-----------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("mode", _to_java_column(col), _enum_to_value(deterministic)) + return _invoke_function_over_columns("isnull", col) @_try_remote_functions -def max(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the maximum value of the expression in a group. - - .. versionadded:: 1.3.0 +def rlike(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the maximum value is computed. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A column that contains the maximum value computed. + true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. See Also -------- - :meth:`pyspark.sql.functions.min` - :meth:`pyspark.sql.functions.avg` - :meth:`pyspark.sql.functions.sum` - - Notes - ----- - - Null values are ignored during the computation. - - NaN values are larger than any other numeric value. + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.regexp_like` + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.ilike` Examples -------- - Example 1: Compute the maximum value of a numeric column - >>> import pyspark.sql.functions as sf - >>> df = spark.range(10) - >>> df.select(sf.max(df.id)).show() - +-------+ - |max(id)| - +-------+ - | 9| - +-------+ + >>> df = spark.createDataFrame([("1a 2b 14m", r"(\d+)")], ["str", "regexp"]) + >>> df.select('*', sf.rlike('str', sf.lit(r'(\d+)'))).show() + +---------+------+-----------------+ + | str|regexp|RLIKE(str, (\d+))| + +---------+------+-----------------+ + |1a 2b 14m| (\d+)| true| + +---------+------+-----------------+ - Example 2: Compute the maximum value of a string column + >>> df.select('*', sf.rlike('str', sf.lit(r'\d{2}b'))).show() + +---------+------+------------------+ + | str|regexp|RLIKE(str, \d{2}b)| + +---------+------+------------------+ + |1a 2b 14m| (\d+)| false| + +---------+------+------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("A",), ("B",), ("C",)], ["value"]) - >>> df.select(sf.max(df.value)).show() - +----------+ - |max(value)| - +----------+ - | C| - +----------+ + >>> df.select('*', sf.rlike("str", sf.col("regexp"))).show() + +---------+------+------------------+ + | str|regexp|RLIKE(str, regexp)| + +---------+------+------------------+ + |1a 2b 14m| (\d+)| true| + +---------+------+------------------+ - Example 3: Compute the maximum value of a column in a grouped DataFrame + >>> df.select('*', sf.rlike("str", "regexp")).show() + +---------+------+------------------+ + | str|regexp|RLIKE(str, regexp)| + +---------+------+------------------+ + |1a 2b 14m| (\d+)| true| + +---------+------+------------------+ + """ + return _invoke_function_over_columns("rlike", str, regexp) - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("A", 1), ("A", 2), ("B", 3), ("B", 4)], ["key", "value"]) - >>> df.groupBy("key").agg(sf.max(df.value)).show() - +---+----------+ - |key|max(value)| - +---+----------+ - | A| 2| - | B| 4| - +---+----------+ - Example 4: Compute the maximum value of multiple columns in a grouped DataFrame +@_try_remote_functions +def regexp(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame( - ... [("A", 1, 2), ("A", 2, 3), ("B", 3, 4), ("B", 4, 5)], ["key", "value1", "value2"]) - >>> df.groupBy("key").agg(sf.max("value1"), sf.max("value2")).show() - +---+-----------+-----------+ - |key|max(value1)|max(value2)| - +---+-----------+-----------+ - | A| 2| 3| - | B| 4| 5| - +---+-----------+-----------+ + .. versionadded:: 3.5.0 - Example 5: Compute the maximum value of a column with null values + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or str + regex pattern to apply. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. + + See Also + -------- + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp_like` + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.ilike` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (None,)], ["value"]) - >>> df.select(sf.max(df.value)).show() - +----------+ - |max(value)| - +----------+ - | 2| - +----------+ + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp('str', sf.lit(r'(\d+)'))).show() + +------------------+ + |REGEXP(str, (\d+))| + +------------------+ + | true| + +------------------+ - Example 6: Compute the maximum value of a column with "NaN" values + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp('str', sf.lit(r'\d{2}b'))).show() + +-------------------+ + |REGEXP(str, \d{2}b)| + +-------------------+ + | false| + +-------------------+ >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1.1,), (float("nan"),), (3.3,)], ["value"]) - >>> df.select(sf.max(df.value)).show() - +----------+ - |max(value)| - +----------+ - | NaN| - +----------+ + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp('str', sf.col("regexp"))).show() + +-------------------+ + |REGEXP(str, regexp)| + +-------------------+ + | true| + +-------------------+ """ - return _invoke_function_over_columns("max", col) + return _invoke_function_over_columns("regexp", str, regexp) @_try_remote_functions -def min(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the minimum value of the expression in a group. - - .. versionadded:: 1.3.0 +def regexp_like(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the minimum value is computed. + str : :class:`~pyspark.sql.Column` or str + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or str + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A column that contains the minimum value computed. + true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. See Also -------- - :meth:`pyspark.sql.functions.max` - :meth:`pyspark.sql.functions.avg` - :meth:`pyspark.sql.functions.sum` + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.ilike` Examples -------- - Example 1: Compute the minimum value of a numeric column - >>> import pyspark.sql.functions as sf - >>> df = spark.range(10) - >>> df.select(sf.min(df.id)).show() - +-------+ - |min(id)| - +-------+ - | 0| - +-------+ + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp_like('str', sf.lit(r'(\d+)'))).show() + +-----------------------+ + |REGEXP_LIKE(str, (\d+))| + +-----------------------+ + | true| + +-----------------------+ - Example 2: Compute the minimum value of a string column + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp_like('str', sf.lit(r'\d{2}b'))).show() + +------------------------+ + |REGEXP_LIKE(str, \d{2}b)| + +------------------------+ + | false| + +------------------------+ >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Alice",), ("Bob",), ("Charlie",)], ["name"]) - >>> df.select(sf.min("name")).show() - +---------+ - |min(name)| - +---------+ - | Alice| - +---------+ + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp_like('str', sf.col("regexp"))).show() + +------------------------+ + |REGEXP_LIKE(str, regexp)| + +------------------------+ + | true| + +------------------------+ + """ + return _invoke_function_over_columns("regexp_like", str, regexp) - Example 3: Compute the minimum value of a column with null values - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1,), (None,), (3,)], ["value"]) - >>> df.select(sf.min("value")).show() - +----------+ - |min(value)| - +----------+ - | 1| - +----------+ +@_try_remote_functions +def like( + str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None +) -> Column: + """ + Returns true if str matches `pattern` with `escape`, + null if any arguments are null, false otherwise. + The default escape character is the '\'. - Example 4: Compute the minimum value of a column in a grouped DataFrame + .. versionadded:: 3.5.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Alice", 1), ("Alice", 2), ("Bob", 3)], ["name", "value"]) - >>> df.groupBy("name").agg(sf.min("value")).show() - +-----+----------+ - | name|min(value)| - +-----+----------+ - |Alice| 1| - | Bob| 3| - +-----+----------+ + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + A string. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or str + A string. The pattern is a string which is matched literally, with + exception to the following special symbols: + _ matches any one character in the input (similar to . in posix regular expressions) + % matches zero or more characters in the input (similar to .* in posix regular + expressions) + Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order + to match "\abc", the pattern should be "\\abc". + When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back + to Spark 1.6 behavior regarding string literal parsing. For example, if the config is + enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. + escapeChar : :class:`~pyspark.sql.Column`, optional + An character added since Spark 3.0. The default escape character is the '\'. + If an escape character precedes a special symbol or another escape character, the + following character is matched literally. It is invalid to escape any other character. + A column that evaluates to a string. - Example 5: Compute the minimum value of a column in a DataFrame with multiple columns + See Also + -------- + :meth:`pyspark.sql.functions.ilike` + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.regexp_like` + + Examples + -------- + >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) + >>> df.select(like(df.a, df.b).alias('r')).collect() + [Row(r=True)] - >>> import pyspark.sql.functions as sf >>> df = spark.createDataFrame( - ... [("Alice", 1, 100), ("Bob", 2, 200), ("Charlie", 3, 300)], - ... ["name", "value1", "value2"]) - >>> df.select(sf.min("value1"), sf.min("value2")).show() - +-----------+-----------+ - |min(value1)|min(value2)| - +-----------+-----------+ - | 1| 100| - +-----------+-----------+ + ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], + ... ['a', 'b'] + ... ) + >>> df.select(like(df.a, df.b, lit('/')).alias('r')).collect() + [Row(r=True)] """ - return _invoke_function_over_columns("min", col) + if escapeChar is not None: + return _invoke_function_over_columns("like", str, pattern, escapeChar) + else: + return _invoke_function_over_columns("like", str, pattern) @_try_remote_functions -def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: +def ilike( + str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None +) -> Column: """ - Returns the value(s) from the `col` parameter that are associated with the maximum value(s) - from the `ord` parameter. This function is often used to find the `col` parameter value - corresponding to the maximum `ord` parameter value within each group when used with groupBy(). + Returns true if str matches `pattern` with `escape` case-insensitively, + null if any arguments are null, false otherwise. + The default escape character is the '\'. - When `k` is specified, returns an array of up to `k` values associated with the top `k` - maximum values from `ord`. + .. versionadded:: 3.5.0 - .. versionadded:: 3.3.0 + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + A string. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or str + A string. The pattern is a string which is matched literally, with + exception to the following special symbols: + _ matches any one character in the input (similar to . in posix regular expressions) + % matches zero or more characters in the input (similar to .* in posix regular + expressions) + Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order + to match "\abc", the pattern should be "\\abc". + When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back + to Spark 1.6 behavior regarding string literal parsing. For example, if the config is + enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. + escapeChar : :class:`~pyspark.sql.Column`, optional + An character added since Spark 3.0. The default escape character is the '\'. + If an escape character precedes a special symbol or another escape character, the + following character is matched literally. It is invalid to escape any other character. + A column that evaluates to a string. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + See Also + -------- + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.regexp_like` - .. versionchanged:: 4.2.0 - Added optional `k` parameter to return top-k values. + Examples + -------- + >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) + >>> df.select(ilike(df.a, df.b).alias('r')).collect() + [Row(r=True)] - Notes - ----- - The function is non-deterministic so the output order can be different for those - associated the same values of `col`. + >>> df = spark.createDataFrame( + ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], + ... ['a', 'b'] + ... ) + >>> df.select(ilike(df.a, df.b, lit('/')).alias('r')).collect() + [Row(r=True)] + """ + if escapeChar is not None: + return _invoke_function_over_columns("ilike", str, pattern, escapeChar) + else: + return _invoke_function_over_columns("ilike", str, pattern) + + +@_try_remote_functions +def isnotnull(col: "ColumnOrName") -> Column: + """ + Returns true if `col` is not null, or false otherwise. + + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column representing the values to be returned. This could be the column instance - or the column name as string. - A column of any type. - ord : :class:`~pyspark.sql.Column` or column name - The column that needs to be maximized. This could be the column instance - or the column name as string. - A column of any orderable type. - k : int, optional - If specified, returns an array of up to `k` values associated with the top `k` - maximum ordering values, sorted in descending order by the ordering column. - Must be a positive integer literal <= 100000. - A column that evaluates to an integer. Must be a constant. - Returns - ------- - :class:`~pyspark.sql.Column` - A column object representing the value from `col` that is associated with - the maximum value from `ord`. If `k` is specified, returns an array of values. + See Also + -------- + :meth:`pyspark.sql.functions.isnan` + :meth:`pyspark.sql.functions.isnull` Examples -------- - Example 1: Using `max_by` with groupBy + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) + >>> df.select('*', sf.isnotnull(df.e)).show() + +----+---------------+ + | e|(e IS NOT NULL)| + +----+---------------+ + |NULL| false| + | 1| true| + +----+---------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.max_by("year", "earnings")).sort("course").show() - +------+----------------------+ - |course|max_by(year, earnings)| - +------+----------------------+ - | Java| 2013| - |dotNET| 2013| - +------+----------------------+ + >>> df.select('*', sf.isnotnull('e')).show() + +----+---------------+ + | e|(e IS NOT NULL)| + +----+---------------+ + |NULL| false| + | 1| true| + +----+---------------+ + """ + return _invoke_function_over_columns("isnotnull", col) - Example 2: Using `max_by` with different data types - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), - ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.max_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|max_by(name, years_in_dept)| - +----------+---------------------------+ - | IT| Charlie| - | Marketing| Anna| - +----------+---------------------------+ +@_try_remote_functions +def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """ + Returns same result as the EQUAL(=) operator for non-null operands, + but returns true if both are null, false if one of them is null. - Example 3: Using `max_by` where `ord` has multiple maximum values + .. versionadded:: 3.5.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), - ... ("Finance", "George", 9), ("Consult", "Henry", 7)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.max_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|max_by(name, years_in_dept)| - +----------+---------------------------+ - | Consult| Henry| - | Finance| George| - +----------+---------------------------+ + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. + col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. - Example 4: Using `max_by` with `k` to get top-k values + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) + >>> df.select('*', sf.equal_null(df.a, df.b)).show() + +----+----+----------------+ + | a| b|equal_null(a, b)| + +----+----+----------------+ + |NULL|NULL| true| + | 1| 9| false| + +----+----+----------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], - ... schema=("x", "y")) - >>> df.select(sf.max_by("x", "y", 2)).show() - +---------------+ - |max_by(x, y, 2)| - +---------------+ - | [b, d]| - +---------------+ + >>> df.select('*', sf.equal_null('a', 'b')).show() + +----+----+----------------+ + | a| b|equal_null(a, b)| + +----+----+----------------+ + |NULL|NULL| true| + | 1| 9| false| + +----+----+----------------+ """ - if k is not None: - return _invoke_function_over_columns("max_by", col, ord, lit(k)) - return _invoke_function_over_columns("max_by", col, ord) + return _invoke_function_over_columns("equal_null", col1, col2) + + +# ---------------------- Sort Functions ---------------------- @_try_remote_functions -def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: +def asc(col: "ColumnOrName") -> Column: """ - Returns the value(s) from the `col` parameter that are associated with the minimum value(s) - from the `ord` parameter. This function is often used to find the `col` parameter value - corresponding to the minimum `ord` parameter value within each group when used with groupBy(). + Returns a sort expression for the target column in ascending order. + This function is used in `sort` and `orderBy` functions. - When `k` is specified, returns an array of up to `k` values associated with the bottom `k` - minimum values from `ord`. - - .. versionadded:: 3.3.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.2.0 - Added optional `k` parameter to return bottom-k values. - - Notes - ----- - The function is non-deterministic so the output order can be different for those - associated the same values of `col`. - Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column representing the values that will be returned. This could be the column instance - or the column name as string. - A column of any type. - ord : :class:`~pyspark.sql.Column` or column name - The column that needs to be minimized. This could be the column instance - or the column name as string. - A column of any orderable type. - k : int, optional - If specified, returns an array of up to `k` values associated with the bottom `k` - minimum ordering values, sorted in ascending order by the ordering column. - Must be a positive integer literal <= 100000. - A column that evaluates to an integer. Must be a constant. + Target column to sort by in the ascending order. Returns ------- :class:`~pyspark.sql.Column` - Column object that represents the value from `col` associated with - the minimum value from `ord`. If `k` is specified, returns an array of values. + The column specifying the sort order. + + See Also + -------- + :meth:`pyspark.sql.functions.asc_nulls_first` + :meth:`pyspark.sql.functions.asc_nulls_last` Examples -------- - Example 1: Using `min_by` with groupBy: + Example 1: Sort DataFrame by 'id' column in ascending order. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.min_by("year", "earnings")).sort("course").show() - +------+----------------------+ - |course|min_by(year, earnings)| - +------+----------------------+ - | Java| 2012| - |dotNET| 2012| - +------+----------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(sf.asc("id")).show() + +---+-----+ + | id|value| + +---+-----+ + | 2| C| + | 3| A| + | 4| B| + +---+-----+ - Example 2: Using `min_by` with different data types: + Example 2: Use `asc` in `orderBy` function to sort the DataFrame. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), - ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.min_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|min_by(name, years_in_dept)| - +----------+---------------------------+ - | IT| Bob| - | Marketing| David| - +----------+---------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.orderBy(sf.asc("value")).show() + +---+-----+ + | id|value| + +---+-----+ + | 3| A| + | 4| B| + | 2| C| + +---+-----+ - Example 3: Using `min_by` where `ord` has multiple minimum values: + Example 3: Combine `asc` with `desc` to sort by multiple columns. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), - ... ("Finance", "George", 9), ("Consult", "Henry", 7)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.min_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|min_by(name, years_in_dept)| - +----------+---------------------------+ - | Consult| Eva| - | Finance| Frank| - +----------+---------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], + ... ['id', 'group', 'value']) + >>> df.sort(sf.asc("group"), sf.desc("value")).show() + +---+-----+-----+ + | id|group|value| + +---+-----+-----+ + | 2| A| 4| + | 3| A| 2| + | 1| B| 3| + +---+-----+-----+ - Example 4: Using `min_by` with `k` to get bottom-k values + Example 4: Implement `asc` from column expression. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], - ... schema=("x", "y")) - >>> df.select(sf.min_by("x", "y", 2)).show() - +---------------+ - |min_by(x, y, 2)| - +---------------+ - | [a, c]| - +---------------+ + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(df.id.asc()).show() + +---+-----+ + | id|value| + +---+-----+ + | 2| C| + | 3| A| + | 4| B| + +---+-----+ """ - if k is not None: - return _invoke_function_over_columns("min_by", col, ord, lit(k)) - return _invoke_function_over_columns("min_by", col, ord) + return col.asc() if isinstance(col, Column) else _invoke_function("asc", col) @_try_remote_functions -def count(col: "ColumnOrName") -> Column: +def desc(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the number of items in a group. + Returns a sort expression for the target column in descending order. + This function is used in `sort` and `orderBy` functions. .. versionadded:: 1.3.0 @@ -1610,71 +1447,83 @@ def count(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + Target column to sort by in the descending order. Returns ------- :class:`~pyspark.sql.Column` - column for computed results. + The column specifying the sort order. See Also -------- - :meth:`pyspark.sql.functions.count_if` + :meth:`pyspark.sql.functions.desc_nulls_first` + :meth:`pyspark.sql.functions.desc_nulls_last` Examples -------- - Example 1: Count all rows in a DataFrame + Example 1: Sort DataFrame by 'id' column in descending order. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), ("a",), ("b",), ("c",)], schema=["alphabets"]) - >>> df.select(sf.count(sf.expr("*"))).show() - +--------+ - |count(1)| - +--------+ - | 4| - +--------+ + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(sf.desc("id")).show() + +---+-----+ + | id|value| + +---+-----+ + | 4| B| + | 3| A| + | 2| C| + +---+-----+ - Example 2: Count non-null values in a specific column + Example 2: Use `desc` in `orderBy` function to sort the DataFrame. >>> from pyspark.sql import functions as sf - >>> df.select(sf.count(df.alphabets)).show() - +----------------+ - |count(alphabets)| - +----------------+ - | 3| - +----------------+ + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.orderBy(sf.desc("value")).show() + +---+-----+ + | id|value| + +---+-----+ + | 2| C| + | 4| B| + | 3| A| + +---+-----+ - Example 3: Count all rows in a DataFrame with multiple columns + Example 3: Combine `asc` with `desc` to sort by multiple columns. >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame( - ... [(1, "apple"), (2, "banana"), (3, None)], schema=["id", "fruit"]) - >>> df.select(sf.count(sf.expr("*"))).show() - +--------+ - |count(1)| - +--------+ - | 3| - +--------+ + ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], + ... ['id', 'group', 'value']) + >>> df.sort(sf.desc("group"), sf.asc("value")).show() + +---+-----+-----+ + | id|group|value| + +---+-----+-----+ + | 1| B| 3| + | 3| A| 2| + | 2| A| 4| + +---+-----+-----+ - Example 4: Count non-null values in multiple columns + Example 4: Implement `desc` from column expression. - >>> from pyspark.sql import functions as sf - >>> df.select(sf.count(df.id), sf.count(df.fruit)).show() - +---------+------------+ - |count(id)|count(fruit)| - +---------+------------+ - | 3| 2| - +---------+------------+ + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(df.id.desc()).show() + +---+-----+ + | id|value| + +---+-----+ + | 4| B| + | 3| A| + | 2| C| + +---+-----+ """ - return _invoke_function_over_columns("count", col) + return col.desc() if isinstance(col, Column) else _invoke_function("desc", col) @_try_remote_functions -def sum(col: "ColumnOrName") -> Column: +def asc_nulls_first(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the sum of all values in the expression. + Sort Function: Returns a sort expression based on the ascending order of the given + column name, and null values return before non-null values. - .. versionadded:: 1.3.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1682,64 +1531,74 @@ def sum(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric or interval. + target column to sort by in the ascending order. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + the column specifying the order. See Also -------- - :meth:`pyspark.sql.functions.min` - :meth:`pyspark.sql.functions.max` - :meth:`pyspark.sql.functions.avg` + :meth:`pyspark.sql.functions.asc` + :meth:`pyspark.sql.functions.asc_nulls_last` Examples -------- - Example 1: Calculating the sum of values in a column + Example 1: Sorting a DataFrame with null values in ascending order >>> from pyspark.sql import functions as sf - >>> df = spark.range(10) - >>> df.select(sf.sum(df["id"])).show() - +-------+ - |sum(id)| - +-------+ - | 45| - +-------+ + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_first(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 2|Alice| + | 1| Bob| + +---+-----+ - Example 2: Using a plus expression together to calculate the sum + Example 2: Sorting a DataFrame with multiple columns, null values in ascending order >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) - >>> df.select(sf.sum(sf.col("A") + sf.col("B"))).show() - +------------+ - |sum((A + B))| - +------------+ - | 10| - +------------+ + >>> df = spark.createDataFrame( + ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.asc_nulls_first(df.name), sf.asc_nulls_first(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 0| NULL| Z| + | 2|Alice| Y| + | 1| Bob| NULL| + +---+-----+-----+ - Example 3: Calculating the summation of ages with None + Example 3: Sorting a DataFrame with null values in ascending order using column name string - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.sum("age")).show() - +--------+ - |sum(age)| - +--------+ - | 6| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_first("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 2|Alice| + | 1| Bob| + +---+-----+ """ - return _invoke_function_over_columns("sum", col) + return ( + col.asc_nulls_first() + if isinstance(col, Column) + else _invoke_function("asc_nulls_first", col) + ) @_try_remote_functions -def avg(col: "ColumnOrName") -> Column: +def asc_nulls_last(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the average of the values in a group. + Sort Function: Returns a sort expression based on the ascending order of the given + column name, and null values appear after non-null values. - .. versionadded:: 1.3.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1747,54 +1606,72 @@ def avg(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric or interval. + target column to sort by in the ascending order. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + the column specifying the order. See Also -------- - :meth:`pyspark.sql.functions.min` - :meth:`pyspark.sql.functions.max` - :meth:`pyspark.sql.functions.sum` + :meth:`pyspark.sql.functions.asc` + :meth:`pyspark.sql.functions.asc_nulls_first` Examples -------- - Example 1: Calculating the average age + Example 1: Sorting a DataFrame with null values in ascending order - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) - >>> df.select(sf.avg("age")).show() - +--------+ - |avg(age)| - +--------+ - | 8.5| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_last(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 2|Alice| + | 1| Bob| + | 0| NULL| + +---+-----+ - Example 2: Calculating the average age with None + Example 2: Sorting a DataFrame with multiple columns, null values in ascending order - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.avg("age")).show() - +--------+ - |avg(age)| - +--------+ - | 3.0| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.asc_nulls_last(df.name), sf.asc_nulls_last(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 2|Alice| Y| + | 1| Bob| NULL| + | 0| NULL| Z| + +---+-----+-----+ + + Example 3: Sorting a DataFrame with null values in ascending order using column name string + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_last("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 2|Alice| + | 1| Bob| + | 0| NULL| + +---+-----+ """ - return _invoke_function_over_columns("avg", col) + return ( + col.asc_nulls_last() if isinstance(col, Column) else _invoke_function("asc_nulls_last", col) + ) @_try_remote_functions -def mean(col: "ColumnOrName") -> Column: +def desc_nulls_first(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the average of the values in a group. - An alias of :func:`avg`. + Sort Function: Returns a sort expression based on the descending order of the given + column name, and null values appear before non-null values. - .. versionadded:: 1.4.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1802,115 +1679,151 @@ def mean(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric or interval. + target column to sort by in the descending order. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + the column specifying the order. + + See Also + -------- + :meth:`pyspark.sql.functions.desc` + :meth:`pyspark.sql.functions.desc_nulls_last` Examples -------- - Example 1: Calculating the average age + Example 1: Sorting a DataFrame with null values in descending order - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) - >>> df.select(sf.mean("age")).show() - +--------+ - |avg(age)| - +--------+ - | 8.5| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_first(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 1| Bob| + | 2|Alice| + +---+-----+ - Example 2: Calculating the average age with None + Example 2: Sorting a DataFrame with multiple columns, null values in descending order - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.mean("age")).show() - +--------+ - |avg(age)| - +--------+ - | 3.0| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.desc_nulls_first(df.name), sf.desc_nulls_first(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 0| NULL| Z| + | 1| Bob| NULL| + | 2|Alice| Y| + +---+-----+-----+ + + Example 3: Sorting a DataFrame with null values in descending order using column name string + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_first("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 1| Bob| + | 2|Alice| + +---+-----+ """ - return _invoke_function_over_columns("mean", col) + return ( + col.desc_nulls_first() + if isinstance(col, Column) + else _invoke_function("desc_nulls_first", col) + ) @_try_remote_functions -def median(col: "ColumnOrName") -> Column: +def desc_nulls_last(col: "ColumnOrName") -> Column: """ - Returns the median of the values in a group. + Sort Function: Returns a sort expression based on the descending order of the given + column name, and null values appear after non-null values. - .. versionadded:: 3.4.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric, interval, or time. + target column to sort by in the descending order. Returns ------- :class:`~pyspark.sql.Column` - the median of the values in a group. - - Notes - ----- - Supports Spark Connect. - - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.approx_percentile` - :meth:`pyspark.sql.functions.percentile_approx` + the column specifying the order. See Also -------- - :meth:`pyspark.sql.functions.approx_percentile` - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.percentile_approx` + :meth:`pyspark.sql.functions.desc` + :meth:`pyspark.sql.functions.desc_nulls_first` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("Java", 2012, 22000), ("dotNET", 2012, 10000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.median("earnings")).show() - +------+----------------+ - |course|median(earnings)| - +------+----------------+ - | Java| 22000.0| - |dotNET| 10000.0| - +------+----------------+ - """ - return _invoke_function_over_columns("median", col) + Example 1: Sorting a DataFrame with null values in descending order + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_last(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 1| Bob| + | 2|Alice| + | 0| NULL| + +---+-----+ -@_try_remote_functions -def sumDistinct(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the sum of distinct values in the expression. + Example 2: Sorting a DataFrame with multiple columns, null values in descending order - .. versionadded:: 1.3.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.desc_nulls_last(df.name), sf.desc_nulls_last(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 1| Bob| NULL| + | 2|Alice| Y| + | 0| NULL| Z| + +---+-----+-----+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Sorting a DataFrame with null values in descending order using column name string - .. deprecated:: 3.2.0 - Use :func:`sum_distinct` instead. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_last("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 1| Bob| + | 2|Alice| + | 0| NULL| + +---+-----+ """ - warnings.warn("Deprecated in 3.2, use sum_distinct instead.", FutureWarning) - return sum_distinct(col) + return ( + col.desc_nulls_last() + if isinstance(col, Column) + else _invoke_function("desc_nulls_last", col) + ) + + +# ---------------------- Mathematical Functions ---------------------- @_try_remote_functions -def sum_distinct(col: "ColumnOrName") -> Column: +def sqrt(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the sum of distinct values in the expression. + Computes the square root of the specified float value. - .. versionadded:: 3.2.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1919,381 +1832,389 @@ def sum_distinct(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + column for computed results. + Returns a column that evaluates to a double. Examples -------- - Example 1: Using sum_distinct function on a column with all distinct values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,)], ["numbers"]) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | 10| - +---------------------+ - - Example 2: Using sum_distinct function on a column with no distinct values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (1,), (1,), (1,)], ["numbers"]) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | 1| - +---------------------+ - - Example 3: Using sum_distinct function on a column with null and duplicate values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), (1,), (1,), (2,)], ["numbers"]) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | 3| - +---------------------+ - - Example 4: Using sum_distinct function on a column with all None values - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, IntegerType - >>> schema = StructType([StructField("numbers", IntegerType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | NULL| - +---------------------+ + >>> spark.sql( + ... "SELECT * FROM VALUES (-1), (0), (1), (4), (NULL) AS TAB(value)" + ... ).select("*", sf.sqrt("value")).show() + +-----+-----------+ + |value|SQRT(value)| + +-----+-----------+ + | -1| NaN| + | 0| 0.0| + | 1| 1.0| + | 4| 2.0| + | NULL| NULL| + +-----+-----------+ """ - return _invoke_function_over_columns("sum_distinct", col) + return _invoke_function_over_columns("sqrt", col) @_try_remote_functions -def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None) -> Column: +def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Aggregate function: returns the concatenation of non-null input values, - separated by the delimiter. + Returns the sum of `left`and `right` and the result is null on overflow. + The acceptable input types are the same with the `+` operator. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a string or binary. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. - A column that evaluates to a string, binary, or null. Must be a constant. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. + left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. + right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- - Example 1: Using listagg function + Example 1: Integer plus Integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.listagg('strings')).show() - +----------------------+ - |listagg(strings, NULL)| - +----------------------+ - | abc| - +----------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(1982, 15), (1990, 2)], ["birth", "age"] + ... ).select("*", sf.try_add("birth", "age")).show() + +-----+---+-------------------+ + |birth|age|try_add(birth, age)| + +-----+---+-------------------+ + | 1982| 15| 1997| + | 1990| 2| 1992| + +-----+---+-------------------+ - Example 2: Using listagg function with a delimiter + Example 2: Date plus Integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.listagg('strings', ', ')).show() - +--------------------+ - |listagg(strings, , )| - +--------------------+ - | a, b, c| - +--------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-09-30')) AS TAB(date)" + ... ).select("*", sf.try_add("date", sf.lit(1))).show() + +----------+----------------+ + | date|try_add(date, 1)| + +----------+----------------+ + |2015-09-30| 2015-10-01| + +----------+----------------+ - Example 3: Using listagg function with a binary column and delimiter + Example 3: Date plus Interval. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) - >>> df.select(sf.listagg('bytes', b'\x42')).show() - +---------------------+ - |listagg(bytes, X'42')| - +---------------------+ - | [01 42 02 42 03]| - +---------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" + ... ).select("*", sf.try_add("date", "itvl")).show() + +----------+-----------------+-------------------+ + | date| itvl|try_add(date, itvl)| + +----------+-----------------+-------------------+ + |2015-09-30|INTERVAL '1' YEAR| 2016-09-30| + +----------+-----------------+-------------------+ - Example 4: Using listagg function on a column with all None values + Example 4: Interval plus Interval. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.listagg('strings')).show() - +----------------------+ - |listagg(strings, NULL)| - +----------------------+ - | NULL| - +----------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" + ... ).select("*", sf.try_add("itvl1", "itvl2")).show() + +-----------------+-----------------+---------------------+ + | itvl1| itvl2|try_add(itvl1, itvl2)| + +-----------------+-----------------+---------------------+ + |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '3' YEAR| + +-----------------+-----------------+---------------------+ + + Example 5: Overflow results in NULL when ANSI mode is on + + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_add(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-------------------------------------------------+ + |try_add(9223372036854775807, 9223372036854775807)| + +-------------------------------------------------+ + | NULL| + +-------------------------------------------------+ """ - if delimiter is None: - return _invoke_function_over_columns("listagg", col) - else: - return _invoke_function_over_columns("listagg", col, lit(delimiter)) + return _invoke_function_over_columns("try_add", left, right) @_try_remote_functions -def listagg_distinct( - col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None -) -> Column: +def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Aggregate function: returns the concatenation of distinct non-null input values, - separated by the delimiter. + Returns `dividend`/`divisor`. It always performs floating point division. Its result is + always null if `divisor` is 0. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. + left : :class:`~pyspark.sql.Column` or column name + dividend. + A column that evaluates to a numeric or interval. + right : :class:`~pyspark.sql.Column` or column name + divisor. + A column that evaluates to a numeric. Examples -------- - Example 1: Using listagg_distinct function - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.listagg_distinct('strings')).show() - +-------------------------------+ - |listagg(DISTINCT strings, NULL)| - +-------------------------------+ - | abc| - +-------------------------------+ - - Example 2: Using listagg_distinct function with a delimiter + Example 1: Integer divided by Integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.listagg_distinct('strings', ', ')).show() - +-----------------------------+ - |listagg(DISTINCT strings, , )| - +-----------------------------+ - | a, b, c| - +-----------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(6000, 15), (1990, 2), (1234, 0)], ["a", "b"] + ... ).select("*", sf.try_divide("a", "b")).show() + +----+---+----------------+ + | a| b|try_divide(a, b)| + +----+---+----------------+ + |6000| 15| 400.0| + |1990| 2| 995.0| + |1234| 0| NULL| + +----+---+----------------+ - Example 3: Using listagg_distinct function with a binary column and delimiter + Example 2: Interval divided by Integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], - ... ['bytes']) - >>> df.select(sf.listagg_distinct('bytes', b'\x42')).show() - +------------------------------+ - |listagg(DISTINCT bytes, X'42')| - +------------------------------+ - | [01 42 02 42 03]| - +------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.range(4).select(sf.make_interval(sf.lit(1)).alias("itvl"), "id") + >>> df.select("*", sf.try_divide("itvl", "id")).show() + +-------+---+--------------------+ + | itvl| id|try_divide(itvl, id)| + +-------+---+--------------------+ + |1 years| 0| NULL| + |1 years| 1| 1 years| + |1 years| 2| 6 months| + |1 years| 3| 4 months| + +-------+---+--------------------+ - Example 4: Using listagg_distinct function on a column with all None values + Example 3: Exception during division, resulting in NULL when ANSI mode is on - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.listagg_distinct('strings')).show() - +-------------------------------+ - |listagg(DISTINCT strings, NULL)| - +-------------------------------+ - | NULL| - +-------------------------------+ + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_divide("id", sf.lit(0))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-----------------+ + |try_divide(id, 0)| + +-----------------+ + | NULL| + +-----------------+ """ - if delimiter is None: - return _invoke_function_over_columns("listagg_distinct", col) - else: - return _invoke_function_over_columns("listagg_distinct", col, lit(delimiter)) + return _invoke_function_over_columns("try_divide", left, right) @_try_remote_functions -def string_agg( - col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None -) -> Column: +def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Aggregate function: returns the concatenation of non-null input values, - separated by the delimiter. - - An alias of :func:`listagg`. + Returns the remainder after `dividend`/`divisor`. Its result is + always null if `divisor` is 0. .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a string or binary. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. - A column that evaluates to a string, binary, or null. Must be a constant. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. + left : :class:`~pyspark.sql.Column` or column name + dividend. + A column that evaluates to a numeric. + right : :class:`~pyspark.sql.Column` or column name + divisor. + A column that evaluates to a numeric. Examples -------- - Example 1: Using string_agg function - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.string_agg('strings')).show() - +-------------------------+ - |string_agg(strings, NULL)| - +-------------------------+ - | abc| - +-------------------------+ - - Example 2: Using string_agg function with a delimiter - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.string_agg('strings', ', ')).show() - +-----------------------+ - |string_agg(strings, , )| - +-----------------------+ - | a, b, c| - +-----------------------+ - - Example 3: Using string_agg function with a binary column and delimiter + Example 1: Integer divided by Integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) - >>> df.select(sf.string_agg('bytes', b'\x42')).show() - +------------------------+ - |string_agg(bytes, X'42')| - +------------------------+ - | [01 42 02 42 03]| - +------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(6000, 15), (3, 2), (1234, 0)], ["a", "b"] + ... ).select("*", sf.try_mod("a", "b")).show() + +----+---+-------------+ + | a| b|try_mod(a, b)| + +----+---+-------------+ + |6000| 15| 0| + | 3| 2| 1| + |1234| 0| NULL| + +----+---+-------------+ - Example 4: Using string_agg function on a column with all None values + Example 2: Exception during division, resulting in NULL when ANSI mode is on - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.string_agg('strings')).show() - +-------------------------+ - |string_agg(strings, NULL)| - +-------------------------+ - | NULL| - +-------------------------+ + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_mod("id", sf.lit(0))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +--------------+ + |try_mod(id, 0)| + +--------------+ + | NULL| + +--------------+ """ - if delimiter is None: - return _invoke_function_over_columns("string_agg", col) - else: - return _invoke_function_over_columns("string_agg", col, lit(delimiter)) + return _invoke_function_over_columns("try_mod", left, right) @_try_remote_functions -def string_agg_distinct( - col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None -) -> Column: +def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Aggregate function: returns the concatenation of distinct non-null input values, - separated by the delimiter. - - An alias of :func:`listagg_distinct`. + Returns `left`*`right` and the result is null on overflow. The acceptable input types are the + same with the `*` operator. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. + left : :class:`~pyspark.sql.Column` or column name + multiplicand. + A column that evaluates to a numeric or interval. + right : :class:`~pyspark.sql.Column` or column name + multiplier. + A column that evaluates to a numeric or interval. Examples -------- - Example 1: Using string_agg_distinct function + Example 1: Integer multiplied by Integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.string_agg_distinct('strings')).show() - +----------------------------------+ - |string_agg(DISTINCT strings, NULL)| - +----------------------------------+ - | abc| - +----------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(6000, 15), (1990, 2)], ["a", "b"] + ... ).select("*", sf.try_multiply("a", "b")).show() + +----+---+------------------+ + | a| b|try_multiply(a, b)| + +----+---+------------------+ + |6000| 15| 90000| + |1990| 2| 3980| + +----+---+------------------+ - Example 2: Using string_agg_distinct function with a delimiter + Example 2: Interval multiplied by Integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.string_agg_distinct('strings', ', ')).show() - +--------------------------------+ - |string_agg(DISTINCT strings, , )| - +--------------------------------+ - | a, b, c| - +--------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.range(6).select(sf.make_interval(sf.col("id"), sf.lit(3)).alias("itvl"), "id") + >>> df.select("*", sf.try_multiply("itvl", "id")).show() + +----------------+---+----------------------+ + | itvl| id|try_multiply(itvl, id)| + +----------------+---+----------------------+ + | 3 months| 0| 0 seconds| + |1 years 3 months| 1| 1 years 3 months| + |2 years 3 months| 2| 4 years 6 months| + |3 years 3 months| 3| 9 years 9 months| + |4 years 3 months| 4| 17 years| + |5 years 3 months| 5| 26 years 3 months| + +----------------+---+----------------------+ - Example 3: Using string_agg_distinct function with a binary column and delimiter + Example 3: Overflow results in NULL when ANSI mode is on - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], - ... ['bytes']) - >>> df.select(sf.string_agg_distinct('bytes', b'\x42')).show() - +---------------------------------+ - |string_agg(DISTINCT bytes, X'42')| - +---------------------------------+ - | [01 42 02 42 03]| - +---------------------------------+ + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_multiply(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +------------------------------------------------------+ + |try_multiply(9223372036854775807, 9223372036854775807)| + +------------------------------------------------------+ + | NULL| + +------------------------------------------------------+ + """ + return _invoke_function_over_columns("try_multiply", left, right) - Example 4: Using string_agg_distinct function on a column with all None values - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.string_agg_distinct('strings')).show() - +----------------------------------+ - |string_agg(DISTINCT strings, NULL)| - +----------------------------------+ - | NULL| - +----------------------------------+ +@_try_remote_functions +def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - if delimiter is None: - return _invoke_function_over_columns("string_agg_distinct", col) - else: - return _invoke_function_over_columns("string_agg_distinct", col, lit(delimiter)) + Returns `left`-`right` and the result is null on overflow. The acceptable input types are the + same with the `-` operator. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. + right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. + + Examples + -------- + Example 1: Integer minus Integer. + + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(1982, 15), (1990, 2)], ["birth", "age"] + ... ).select("*", sf.try_subtract("birth", "age")).show() + +-----+---+------------------------+ + |birth|age|try_subtract(birth, age)| + +-----+---+------------------------+ + | 1982| 15| 1967| + | 1990| 2| 1988| + +-----+---+------------------------+ + + Example 2: Date minus Integer. + + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-10-01')) AS TAB(date)" + ... ).select("*", sf.try_subtract("date", sf.lit(1))).show() + +----------+---------------------+ + | date|try_subtract(date, 1)| + +----------+---------------------+ + |2015-10-01| 2015-09-30| + +----------+---------------------+ + + Example 3: Date minus Interval. + + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" + ... ).select("*", sf.try_subtract("date", "itvl")).show() + +----------+-----------------+------------------------+ + | date| itvl|try_subtract(date, itvl)| + +----------+-----------------+------------------------+ + |2015-09-30|INTERVAL '1' YEAR| 2014-09-30| + +----------+-----------------+------------------------+ + + Example 4: Interval minus Interval. + + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" + ... ).select("*", sf.try_subtract("itvl1", "itvl2")).show() + +-----------------+-----------------+--------------------------+ + | itvl1| itvl2|try_subtract(itvl1, itvl2)| + +-----------------+-----------------+--------------------------+ + |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '-1' YEAR| + +-----------------+-----------------+--------------------------+ + + Example 5: Overflow results in NULL when ANSI mode is on + + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_subtract(sf.lit(-sys.maxsize), sf.lit(sys.maxsize))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-------------------------------------------------------+ + |try_subtract(-9223372036854775807, 9223372036854775807)| + +-------------------------------------------------------+ + | NULL| + +-------------------------------------------------------+ + """ + return _invoke_function_over_columns("try_subtract", left, right) @_try_remote_functions -def product(col: "ColumnOrName") -> Column: +def abs(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the product of the values in a group. + Mathematical Function: Computes the absolute value of the given column or expression. - .. versionadded:: 3.2.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -2301,27 +2222,59 @@ def product(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column containing values to be multiplied together + The target column or expression to compute the absolute value on. + A column that evaluates to a numeric or interval. Returns ------- - :class:`~pyspark.sql.Column` or column name - the column for computed results. + :class:`~pyspark.sql.Column` + A new column object representing the absolute value of the input. + Returns a column of the same type as the input. Examples -------- + Example 1: Compute the absolute value of a long column + >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT id % 3 AS mod3, id AS value FROM RANGE(10)") - >>> df.groupBy('mod3').agg(sf.product('value')).orderBy('mod3').show() - +----+--------------+ - |mod3|product(value)| - +----+--------------+ - | 0| 0.0| - | 1| 28.0| - | 2| 80.0| - +----+--------------+ + >>> df = spark.createDataFrame([(-1,), (-2,), (-3,), (None,)], ["value"]) + >>> df.select("*", sf.abs(df.value)).show() + +-----+----------+ + |value|abs(value)| + +-----+----------+ + | -1| 1| + | -2| 2| + | -3| 3| + | NULL| NULL| + +-----+----------+ + + Example 2: Compute the absolute value of a double column + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(-1.5,), (-2.5,), (None,), (float("nan"),)], ["value"]) + >>> df.select("*", sf.abs(df.value)).show() + +-----+----------+ + |value|abs(value)| + +-----+----------+ + | -1.5| 1.5| + | -2.5| 2.5| + | NULL| NULL| + | NaN| NaN| + +-----+----------+ + + Example 3: Compute the absolute value of an expression + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 1), (2, -2), (3, 3)], ["id", "value"]) + >>> df.select("*", sf.abs(df.id - df.value)).show() + +---+-----+-----------------+ + | id|value|abs((id - value))| + +---+-----+-----------------+ + | 1| 1| 0| + | 2| -2| 4| + | 3| 3| 0| + +---+-----+-----------------+ """ - return _invoke_function_over_columns("product", col) + return _invoke_function_over_columns("abs", col) @_try_remote_functions @@ -3920,28 +3873,12 @@ def toRadians(col: "ColumnOrName") -> Column: @_try_remote_functions -def bitwiseNOT(col: "ColumnOrName") -> Column: - """ - Computes bitwise not. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. deprecated:: 3.2.0 - Use :func:`bitwise_not` instead. - """ - warnings.warn("Deprecated in 3.2, use bitwise_not instead.", FutureWarning) - return bitwise_not(col) - - -@_try_remote_functions -def bitwise_not(col: "ColumnOrName") -> Column: +def degrees(col: "ColumnOrName") -> Column: """ - Computes bitwise not. + Converts an angle measured in radians to an approximately equivalent angle + measured in degrees. - .. versionadded:: 3.2.0 + .. versionadded:: 2.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -3949,505 +3886,456 @@ def bitwise_not(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + angle in degrees, as if computed by `java.lang.Math.toDegrees()` + Returns a column that evaluates to a double. + + See Also + -------- + :meth:`pyspark.sql.functions.radians` Examples -------- >>> from pyspark.sql import functions as sf >>> spark.sql( - ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" - ... ).select("*", sf.bitwise_not("value")).show() - +-----+------+ - |value|~value| - +-----+------+ - | 0| -1| - | 1| -2| - | 2| -3| - | 3| -4| - | NULL| NULL| - +-----+------+ + ... "SELECT * FROM VALUES (0.0), (PI()), (PI() / 2), (PI() / 4) AS TAB(value)" + ... ).select("*", sf.degrees("value")).show() + +------------------+--------------+ + | value|DEGREES(value)| + +------------------+--------------+ + | 0.0| 0.0| + | 3.141592653589...| 180.0| + |1.5707963267948...| 90.0| + |0.7853981633974...| 45.0| + +------------------+--------------+ """ - return _invoke_function_over_columns("bitwise_not", col) + return _invoke_function_over_columns("degrees", col) @_try_remote_functions -def bit_count(col: "ColumnOrName") -> Column: +def radians(col: "ColumnOrName") -> Column: """ - Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer, - or NULL if the argument is NULL. + Converts an angle measured in degrees to an approximately equivalent angle + measured in radians. - .. versionadded:: 3.5.0 + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral or boolean. + angle in degrees. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - the number of bits that are set in the argument expr as an unsigned 64-bit integer, - or NULL if the argument is NULL. - Returns a column that evaluates to an integer. + angle in radians, as if computed by `java.lang.Math.toRadians()` + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.bit_get` + :meth:`pyspark.sql.functions.degrees` Examples -------- >>> from pyspark.sql import functions as sf >>> spark.sql( - ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" - ... ).select("*", sf.bit_count("value")).show() - +-----+----------------+ - |value|bit_count(value)| - +-----+----------------+ - | 0| 0| - | 1| 1| - | 2| 1| - | 3| 2| - | NULL| NULL| - +-----+----------------+ + ... "SELECT * FROM VALUES (180), (90), (45), (0) AS TAB(value)" + ... ).select("*", sf.radians("value")).show() + +-----+------------------+ + |value| RADIANS(value)| + +-----+------------------+ + | 180| 3.141592653589...| + | 90|1.5707963267948...| + | 45|0.7853981633974...| + | 0| 0.0| + +-----+------------------+ """ - return _invoke_function_over_columns("bit_count", col) + return _invoke_function_over_columns("radians", col) @_try_remote_functions -def bit_get(col: "ColumnOrName", pos: "ColumnOrName") -> Column: +def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: """ - Returns the value of the bit (0 or 1) at the specified position. - The positions are numbered from right to left, starting at zero. - The position argument cannot be negative. + Compute the angle in radians between the positive x-axis of a plane + and the point given by the coordinates - .. versionadded:: 3.5.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. - pos : :class:`~pyspark.sql.Column` or column name - The positions are numbered from right to left, starting at zero. - A column that evaluates to an integer. + col1 : :class:`~pyspark.sql.Column`, column name or float + coordinate on y-axis. + A column that evaluates to a double. + col2 : :class:`~pyspark.sql.Column`, column name or float + coordinate on x-axis. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - the value of the bit (0 or 1) at the specified position. - Returns a column that evaluates to a byte. + the `theta` component of the point + (`r`, `theta`) + in polar coordinates that corresponds to the point + (`x`, `y`) in Cartesian coordinates, + as if computed by `java.lang.Math.atan2()` + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.bit_count` - :meth:`pyspark.sql.functions.getbit` + :meth:`pyspark.sql.functions.atan` + :meth:`pyspark.sql.functions.hypot` Examples -------- - Example 1: Get the bit with a literal position - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[2],[3],[None]], ["value"]) - >>> df.select("*", sf.bit_get("value", sf.lit(1))).show() - +-----+-----------------+ - |value|bit_get(value, 1)| - +-----+-----------------+ - | 1| 0| - | 2| 1| - | 3| 1| - | NULL| NULL| - +-----+-----------------+ - - Example 2: Get the bit with a column position - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) - >>> df.select("*", sf.bit_get(df.value, "pos")).show() - +-----+----+-------------------+ - |value| pos|bit_get(value, pos)| - +-----+----+-------------------+ - | 1| 2| 0| - | 2| 1| 1| - | 3|NULL| NULL| - | NULL| 1| NULL| - +-----+----+-------------------+ + >>> spark.range(1).select(sf.atan2(sf.lit(1), sf.lit(2))).show() + +------------------+ + | ATAN2(1, 2)| + +------------------+ + |0.4636476090008...| + +------------------+ """ - return _invoke_function_over_columns("bit_get", col, pos) + return _invoke_binary_math_function("atan2", col1, col2) @_try_remote_functions -def getbit(col: "ColumnOrName", pos: "ColumnOrName") -> Column: +def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: """ - Returns the value of the bit (0 or 1) at the specified position. - The positions are numbered from right to left, starting at zero. - The position argument cannot be negative. + Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow. - .. versionadded:: 3.5.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. - pos : :class:`~pyspark.sql.Column` or column name - The positions are numbered from right to left, starting at zero. - A column that evaluates to an integer. + col1 : :class:`~pyspark.sql.Column`, column name or float + a leg. + A column that evaluates to a double. + col2 : :class:`~pyspark.sql.Column`, column name or float + b leg. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - the value of the bit (0 or 1) at the specified position. - Returns a column that evaluates to a byte. - - See Also - -------- - :meth:`pyspark.sql.functions.bit_get` - :meth:`pyspark.sql.functions.bit_count` + length of the hypotenuse. + Returns a column that evaluates to a double. Examples -------- - Example 1: Get the bit with a literal position - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[1], [2], [3], [None]], ["value"] - ... ).select("*", sf.getbit("value", sf.lit(1))).show() - +-----+----------------+ - |value|getbit(value, 1)| - +-----+----------------+ - | 1| 0| - | 2| 1| - | 3| 1| - | NULL| NULL| - +-----+----------------+ - - Example 2: Get the bit with a column position - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) - >>> df.select("*", sf.getbit(df.value, "pos")).show() - +-----+----+------------------+ - |value| pos|getbit(value, pos)| - +-----+----+------------------+ - | 1| 2| 0| - | 2| 1| 1| - | 3|NULL| NULL| - | NULL| 1| NULL| - +-----+----+------------------+ + >>> spark.range(1).select(sf.hypot(sf.lit(1), sf.lit(2))).show() + +----------------+ + | HYPOT(1, 2)| + +----------------+ + |2.23606797749...| + +----------------+ """ - return _invoke_function_over_columns("getbit", col, pos) + return _invoke_binary_math_function("hypot", col1, col2) @_try_remote_functions -def asc_nulls_first(col: "ColumnOrName") -> Column: +def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: """ - Sort Function: Returns a sort expression based on the ascending order of the given - column name, and null values return before non-null values. + Returns the value of the first argument raised to the power of the second argument. - .. versionadded:: 2.4.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the ascending order. + col1 : :class:`~pyspark.sql.Column`, column name or float + the base number. + A column that evaluates to a double. + col2 : :class:`~pyspark.sql.Column`, column name or float + the exponent number. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. - - See Also - -------- - :meth:`pyspark.sql.functions.asc` - :meth:`pyspark.sql.functions.asc_nulls_last` + the base rased to the power the argument. + Returns a column that evaluates to a double. Examples -------- - Example 1: Sorting a DataFrame with null values in ascending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_first(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 2|Alice| - | 1| Bob| - +---+-----+ - - Example 2: Sorting a DataFrame with multiple columns, null values in ascending order - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.asc_nulls_first(df.name), sf.asc_nulls_first(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 0| NULL| Z| - | 2|Alice| Y| - | 1| Bob| NULL| - +---+-----+-----+ + >>> spark.range(5).select("*", sf.pow("id", 2)).show() + +---+------------+ + | id|POWER(id, 2)| + +---+------------+ + | 0| 0.0| + | 1| 1.0| + | 2| 4.0| + | 3| 9.0| + | 4| 16.0| + +---+------------+ + """ + return _invoke_binary_math_function("pow", col1, col2) - Example 3: Sorting a DataFrame with null values in ascending order using column name string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_first("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 2|Alice| - | 1| Bob| - +---+-----+ - """ - return ( - col.asc_nulls_first() - if isinstance(col, Column) - else _invoke_function("asc_nulls_first", col) - ) +power = pow @_try_remote_functions -def asc_nulls_last(col: "ColumnOrName") -> Column: +def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", float]) -> Column: """ - Sort Function: Returns a sort expression based on the ascending order of the given - column name, and null values appear after non-null values. - - .. versionadded:: 2.4.0 + Returns the positive value of dividend mod divisor. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the ascending order. + dividend : :class:`~pyspark.sql.Column`, column name or float + the column that contains dividend, or the specified dividend value. + A column that evaluates to a numeric. + divisor : :class:`~pyspark.sql.Column`, column name or float + the column that contains divisor, or the specified divisor value. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. - - See Also - -------- - :meth:`pyspark.sql.functions.asc` - :meth:`pyspark.sql.functions.asc_nulls_first` + positive value of dividend mod divisor. + Returns a column of the same type as the input. + + Notes + ----- + Supports Spark Connect. Examples -------- - Example 1: Sorting a DataFrame with null values in ascending order - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_last(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 2|Alice| - | 1| Bob| - | 0| NULL| - +---+-----+ + >>> df = spark.createDataFrame([ + ... (1.0, float('nan')), (float('nan'), 2.0), (10.0, 3.0), + ... (float('nan'), float('nan')), (-3.0, 4.0), (-10.0, 3.0), + ... (-5.0, -6.0), (7.0, -8.0), (1.0, 2.0)], + ... ("a", "b")) + >>> df.select("*", sf.pmod("a", "b")).show() + +-----+----+----------+ + | a| b|pmod(a, b)| + +-----+----+----------+ + | 1.0| NaN| NaN| + | NaN| 2.0| NaN| + | 10.0| 3.0| 1.0| + | NaN| NaN| NaN| + | -3.0| 4.0| 1.0| + |-10.0| 3.0| 2.0| + | -5.0|-6.0| -5.0| + | 7.0|-8.0| 7.0| + | 1.0| 2.0| 1.0| + +-----+----+----------+ + """ + return _invoke_binary_math_function("pmod", dividend, divisor) - Example 2: Sorting a DataFrame with multiple columns, null values in ascending order - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.asc_nulls_last(df.name), sf.asc_nulls_last(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 2|Alice| Y| - | 1| Bob| NULL| - | 0| NULL| Z| - +---+-----+-----+ +@_try_remote_functions +def width_bucket( + v: "ColumnOrName", + min: "ColumnOrName", + max: "ColumnOrName", + numBucket: Union["ColumnOrName", int], +) -> Column: + """ + Returns the bucket number into which the value of this expression would fall + after being evaluated. Note that input arguments must follow conditions listed below; + otherwise, the method will return null. - Example 3: Sorting a DataFrame with null values in ascending order using column name string + .. versionadded:: 3.5.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or column name + value to compute a bucket number in the histogram. + A column that evaluates to a double or interval. + min : :class:`~pyspark.sql.Column` or column name + minimum value of the histogram. + A column that evaluates to a double or interval. + max : :class:`~pyspark.sql.Column` or column name + maximum value of the histogram. + A column that evaluates to a double or interval. + numBucket : :class:`~pyspark.sql.Column`, column name or int + the number of buckets. + A column that evaluates to a long. + + Returns + ------- + :class:`~pyspark.sql.Column` + the bucket number into which the value would fall after being evaluated + Returns a column that evaluates to a long. + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_last("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 2|Alice| - | 1| Bob| - | 0| NULL| - +---+-----+ + >>> df = spark.createDataFrame([ + ... (5.3, 0.2, 10.6, 5), + ... (-2.1, 1.3, 3.4, 3), + ... (8.1, 0.0, 5.7, 4), + ... (-0.9, 5.2, 0.5, 2)], + ... ['v', 'min', 'max', 'n']) + >>> df.select("*", sf.width_bucket('v', 'min', 'max', 'n')).show() + +----+---+----+---+----------------------------+ + | v|min| max| n|width_bucket(v, min, max, n)| + +----+---+----+---+----------------------------+ + | 5.3|0.2|10.6| 5| 3| + |-2.1|1.3| 3.4| 3| 0| + | 8.1|0.0| 5.7| 4| 5| + |-0.9|5.2| 0.5| 2| 3| + +----+---+----+---+----------------------------+ """ - return ( - col.asc_nulls_last() if isinstance(col, Column) else _invoke_function("asc_nulls_last", col) - ) + numBucket = _enum_to_value(numBucket) + numBucket = lit(numBucket) if isinstance(numBucket, int) else numBucket + return _invoke_function_over_columns("width_bucket", v, min, max, numBucket) @_try_remote_functions -def desc_nulls_first(col: "ColumnOrName") -> Column: - """ - Sort Function: Returns a sort expression based on the descending order of the given - column name, and null values appear before non-null values. +def rand(seed: Optional[int] = None) -> Column: + """Generates a random column with independent and identically distributed (i.i.d.) samples + uniformly distributed in [0.0, 1.0). - .. versionadded:: 2.4.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Notes + ----- + The function is non-deterministic in general case. + Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the descending order. + seed : int, optional + Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. + A column of random values. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.desc` - :meth:`pyspark.sql.functions.desc_nulls_last` + :meth:`pyspark.sql.functions.randn` + :meth:`pyspark.sql.functions.randstr` + :meth:`pyspark.sql.functions.uniform` Examples -------- - Example 1: Sorting a DataFrame with null values in descending order + Example 1: Generate a random column without a seed >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_first(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 1| Bob| - | 2|Alice| - +---+-----+ + >>> spark.range(0, 2, 1, 1).select("*", sf.rand()).show() # doctest: +SKIP + +---+-------------------------+ + | id|rand(-158884697681280011)| + +---+-------------------------+ + | 0| 0.9253464547887...| + | 1| 0.6533254118758...| + +---+-------------------------+ - Example 2: Sorting a DataFrame with multiple columns, null values in descending order + Example 2: Generate a random column with a specific seed - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.desc_nulls_first(df.name), sf.desc_nulls_first(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 0| NULL| Z| - | 1| Bob| NULL| - | 2|Alice| Y| - +---+-----+-----+ + >>> spark.range(0, 2, 1, 1).select("*", sf.rand(seed=42)).show() + +---+------------------+ + | id| rand(42)| + +---+------------------+ + | 0| 0.619189370225...| + | 1|0.5096018842446...| + +---+------------------+ + """ + if seed is not None: + return _invoke_function("rand", _enum_to_value(seed)) + else: + return _invoke_function("rand") - Example 3: Sorting a DataFrame with null values in descending order using column name string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_first("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 1| Bob| - | 2|Alice| - +---+-----+ - """ - return ( - col.desc_nulls_first() - if isinstance(col, Column) - else _invoke_function("desc_nulls_first", col) - ) +random = rand @_try_remote_functions -def desc_nulls_last(col: "ColumnOrName") -> Column: - """ - Sort Function: Returns a sort expression based on the descending order of the given - column name, and null values appear after non-null values. +def randn(seed: Optional[int] = None) -> Column: + """Generates a random column with independent and identically distributed (i.i.d.) samples + from the standard normal distribution. - .. versionadded:: 2.4.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Notes + ----- + The function is non-deterministic in general case. + Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the descending order. + seed : int (default: None) + Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. + A column of random values. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.desc` - :meth:`pyspark.sql.functions.desc_nulls_first` + :meth:`pyspark.sql.functions.rand` + :meth:`pyspark.sql.functions.randstr` + :meth:`pyspark.sql.functions.uniform` Examples -------- - Example 1: Sorting a DataFrame with null values in descending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_last(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 1| Bob| - | 2|Alice| - | 0| NULL| - +---+-----+ - - Example 2: Sorting a DataFrame with multiple columns, null values in descending order + Example 1: Generate a random column without a seed >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.desc_nulls_last(df.name), sf.desc_nulls_last(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 1| Bob| NULL| - | 2|Alice| Y| - | 0| NULL| Z| - +---+-----+-----+ + >>> spark.range(0, 2, 1, 1).select("*", sf.randn()).show() # doctest: +SKIP + +---+--------------------------+ + | id|randn(3968742514375399317)| + +---+--------------------------+ + | 0| -0.47968645355788...| + | 1| -0.4950952457305...| + +---+--------------------------+ - Example 3: Sorting a DataFrame with null values in descending order using column name string + Example 2: Generate a random column with a specific seed - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_last("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 1| Bob| - | 2|Alice| - | 0| NULL| - +---+-----+ + >>> spark.range(0, 2, 1, 1).select("*", sf.randn(seed=42)).show() + +---+------------------+ + | id| randn(42)| + +---+------------------+ + | 0| 2.384479054241...| + | 1|0.1920934041293...| + +---+------------------+ """ - return ( - col.desc_nulls_last() - if isinstance(col, Column) - else _invoke_function("desc_nulls_last", col) - ) + if seed is not None: + return _invoke_function("randn", _enum_to_value(seed)) + else: + return _invoke_function("randn") @_try_remote_functions -def stddev(col: "ColumnOrName") -> Column: +def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: """ - Aggregate function: alias for stddev_samp. + Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 + or at integral part when `scale` < 0. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -4455,85 +4343,119 @@ def stddev(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + The target column or column name to compute the round on. A column that evaluates to a numeric. + scale : :class:`~pyspark.sql.Column` or int, optional + An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. - See Also - -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev_pop` - :meth:`pyspark.sql.functions.stddev_samp` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.skewness` - :meth:`pyspark.sql.functions.kurtosis` + .. versionchanged:: 4.0.0 + Support Column type. Returns ------- :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. + A column for the rounded value. + Returns a column of the same type as the input. Examples -------- + Example 1: Compute the rounded of a column value + >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.stddev("id")).show() - +------------------+ - | stddev(id)| - +------------------+ - |1.8708286933869...| - +------------------+ + >>> spark.range(1).select(sf.round(sf.lit(2.5))).show() + +-------------+ + |round(2.5, 0)| + +-------------+ + | 3.0| + +-------------+ + + Example 2: Compute the rounded of a column value with a specified scale + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.round(sf.lit(2.1267), sf.lit(2))).show() + +----------------+ + |round(2.1267, 2)| + +----------------+ + | 2.13| + +----------------+ """ - return _invoke_function_over_columns("stddev", col) + if scale is None: + return _invoke_function_over_columns("round", col) + else: + scale = _enum_to_value(scale) + scale = lit(scale) if isinstance(scale, int) else scale + return _invoke_function_over_columns("round", col, scale) @_try_remote_functions -def std(col: "ColumnOrName") -> Column: +def truncate(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: """ - Aggregate function: alias for stddev_samp. + Truncate the given value toward zero to `scale` decimal places when `scale` >= 0, + or to the left of the decimal point when `scale` < 0. `scale` defaults to 0. - .. versionadded:: 3.5.0 + Unlike :func:`round`, the result is always rounded toward zero, and unlike :func:`floor` + negative values are not rounded toward negative infinity. + + .. versionadded:: 4.4.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + The target column or column name to truncate. A column that evaluates to a numeric. + scale : :class:`~pyspark.sql.Column` or int, optional + An optional parameter to control the number of decimal places to keep. + A column that evaluates to an integer. Must be a constant. Defaults to 0. Returns ------- :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. + A column for the truncated value, of the same type as the input, except that a decimal + input may return a decimal of different precision and scale. See Also -------- - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.stddev_pop` - :meth:`pyspark.sql.functions.stddev_samp` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.skewness` - :meth:`pyspark.sql.functions.kurtosis` + :meth:`pyspark.sql.functions.round` + :meth:`pyspark.sql.functions.trunc` + :meth:`pyspark.sql.functions.floor` + :meth:`pyspark.sql.functions.ceil` Examples -------- + Example 1: Truncate toward zero to a given number of decimal places + >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.std("id")).show() - +------------------+ - | std(id)| - +------------------+ - |1.8708286933869...| - +------------------+ + >>> spark.range(1).select(sf.truncate(sf.lit(15.79), sf.lit(1)).alias("r")).collect() + [Row(r=15.7)] + + Example 2: Truncation rounds toward zero for negative values + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.truncate(sf.lit(-2.99), sf.lit(0)).alias("r")).collect() + [Row(r=-2.0)] + + Example 3: The scale argument defaults to 0 when omitted + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.truncate(sf.lit(1234.5678)).alias("r")).collect() + [Row(r=1234.0)] """ - return _invoke_function_over_columns("std", col) + if scale is None: + return _invoke_function_over_columns("truncate", col) + else: + scale = _enum_to_value(scale) + scale = lit(scale) if isinstance(scale, int) else scale + return _invoke_function_over_columns("truncate", col, scale) @_try_remote_functions -def stddev_samp(col: "ColumnOrName") -> Column: +def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: """ - Aggregate function: returns the unbiased sample standard deviation of - the expression in a group. + Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 + or at integral part when `scale` < 0. - .. versionadded:: 1.6.0 + .. versionadded:: 2.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -4541,1479 +4463,1456 @@ def stddev_samp(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + The target column or column name to compute the round on. A column that evaluates to a numeric. + scale : :class:`~pyspark.sql.Column` or int, optional + An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. + + .. versionchanged:: 4.0.0 + Support Column type. Returns ------- :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. - - See Also - -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.stddev_pop` - :meth:`pyspark.sql.functions.var_samp` + A column for the rounded value. + Returns a column of the same type as the input. Examples -------- + Example 1: Compute the rounded of a column value + >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.stddev_samp("id")).show() - +------------------+ - | stddev_samp(id)| - +------------------+ - |1.8708286933869...| - +------------------+ + >>> spark.range(1).select(sf.bround(sf.lit(2.5))).show() + +--------------+ + |bround(2.5, 0)| + +--------------+ + | 2.0| + +--------------+ + + Example 2: Compute the rounded of a column value with a specified scale + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.bround(sf.lit(2.1267), sf.lit(2))).show() + +-----------------+ + |bround(2.1267, 2)| + +-----------------+ + | 2.13| + +-----------------+ """ - return _invoke_function_over_columns("stddev_samp", col) + if scale is None: + return _invoke_function_over_columns("bround", col) + else: + scale = _enum_to_value(scale) + scale = lit(scale) if isinstance(scale, int) else scale + return _invoke_function_over_columns("bround", col, scale) @_try_remote_functions -def stddev_pop(col: "ColumnOrName") -> Column: +def greatest(*cols: "ColumnOrName") -> Column: """ - Aggregate function: returns population standard deviation of - the expression in a group. + Returns the greatest value of the list of column names, skipping null values. + This function takes at least 2 parameters. It will return null if all parameters are null. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + cols: :class:`~pyspark.sql.Column` or column name + columns to check for greatest value. + Each a column of any orderable type. Returns ------- :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. + greatest value. + Returns a column of the same type as the input. See Also -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.stddev_samp` - :meth:`pyspark.sql.functions.var_pop` + :meth:`pyspark.sql.functions.least` Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.stddev_pop("id")).show() - +-----------------+ - | stddev_pop(id)| - +-----------------+ - |1.707825127659...| - +-----------------+ + >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) + >>> df.select("*", sf.greatest(df.a, "b", df.c)).show() + +---+---+---+-----------------+ + | a| b| c|greatest(a, b, c)| + +---+---+---+-----------------+ + | 1| 4| 3| 4| + +---+---+---+-----------------+ """ - return _invoke_function_over_columns("stddev_pop", col) + if len(cols) < 2: + raise PySparkValueError( + errorClass="WRONG_NUM_COLUMNS", + messageParameters={"func_name": "greatest", "num_cols": "2"}, + ) + return _invoke_function_over_seq_of_columns("greatest", cols) @_try_remote_functions -def variance(col: "ColumnOrName") -> Column: +def least(*cols: "ColumnOrName") -> Column: """ - Aggregate function: alias for var_samp + Returns the least value of the list of column names, skipping null values. + This function takes at least 2 parameters. It will return null if all parameters are null. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + cols : :class:`~pyspark.sql.Column` or column name + column names or columns to be compared + Each a column of any orderable type. Returns ------- :class:`~pyspark.sql.Column` - variance of given column. + least value. + Returns a column of the same type as the input. See Also -------- - :meth:`pyspark.sql.functions.var_pop` - :meth:`pyspark.sql.functions.var_samp` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.skewness` - :meth:`pyspark.sql.functions.kurtosis` - :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.greatest` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.range(6) - >>> df.select(sf.variance(df.id)).show() - +------------+ - |variance(id)| - +------------+ - | 3.5| - +------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) + >>> df.select("*", sf.least(df.a, "b", df.c)).show() + +---+---+---+--------------+ + | a| b| c|least(a, b, c)| + +---+---+---+--------------+ + | 1| 4| 3| 1| + +---+---+---+--------------+ """ - return _invoke_function_over_columns("variance", col) + if len(cols) < 2: + raise PySparkValueError( + errorClass="WRONG_NUM_COLUMNS", + messageParameters={"func_name": "least", "num_cols": "2"}, + ) + return _invoke_function_over_seq_of_columns("least", cols) + + +@overload +def log(arg1: "ColumnOrName") -> Column: ... + + +@overload +def log(arg1: float, arg2: "ColumnOrName") -> Column: ... @_try_remote_functions -def var_samp(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the unbiased sample variance of - the values in a group. +def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = None) -> Column: + """Returns the first argument-based logarithm of the second argument. - .. versionadded:: 1.6.0 + If there is only one argument, then this takes the natural logarithm of the argument. + + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + arg1 : :class:`~pyspark.sql.Column`, str or float + base number or actual number (in this case base is `e`). + A column that evaluates to a double. + arg2 : :class:`~pyspark.sql.Column`, str or float, optional + number to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - variance of given column. + logariphm of given value. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.var_pop` - :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.ln` Examples -------- + Example 1: Specify both base number and the input value + >>> from pyspark.sql import functions as sf - >>> df = spark.range(6) - >>> df.select(sf.var_samp(df.id)).show() - +------------+ - |var_samp(id)| - +------------+ - | 3.5| - +------------+ - """ - return _invoke_function_over_columns("var_samp", col) + >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") + >>> df.select("*", sf.log(2.0, df.value)).show() + +-----+---------------+ + |value|LOG(2.0, value)| + +-----+---------------+ + | 1| 0.0| + | 2| 1.0| + | 4| 2.0| + +-----+---------------+ + Example 2: Return NULL for invalid input values -@_try_remote_functions -def var_pop(col: "ColumnOrName") -> Column: + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (0), (-1), (NULL) AS t(value)") + >>> df.select("*", sf.log(3.0, df.value)).show() + +-----+------------------+ + |value| LOG(3.0, value)| + +-----+------------------+ + | 1| 0.0| + | 2|0.6309297535714...| + | 0| NULL| + | -1| NULL| + | NULL| NULL| + +-----+------------------+ + + Example 3: Specify only the input value (Natural logarithm) + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") + >>> df.select("*", sf.log(df.value)).show() + +-----+------------------+ + |value| ln(value)| + +-----+------------------+ + | 1| 0.0| + | 2|0.6931471805599...| + | 4|1.3862943611198...| + +-----+------------------+ """ - Aggregate function: returns the population variance of the values in a group. + from pyspark.sql.classic.column import _to_java_column - .. versionadded:: 1.6.0 + if arg2 is None: + return _invoke_function_over_columns("log", cast("ColumnOrName", arg1)) + else: + return _invoke_function("log", _enum_to_value(arg1), _to_java_column(arg2)) - .. versionchanged:: 3.4.0 - Supports Spark Connect. + +@_try_remote_functions +def ln(col: "ColumnOrName") -> Column: + """Returns the natural logarithm of the argument. + + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - variance of given column. + natural logarithm of given value. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.var_samp` - :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.log` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.range(6) - >>> df.select(sf.var_pop(df.id)).show() - +------------------+ - | var_pop(id)| - +------------------+ - |2.9166666666666...| - +------------------+ + >>> spark.range(10).select("*", sf.ln('id')).show() + +---+------------------+ + | id| ln(id)| + +---+------------------+ + | 0| NULL| + | 1| 0.0| + | 2|0.6931471805599...| + | 3|1.0986122886681...| + | 4|1.3862943611198...| + | 5|1.6094379124341...| + | 6| 1.791759469228...| + | 7|1.9459101490553...| + | 8|2.0794415416798...| + | 9|2.1972245773362...| + +---+------------------+ """ - return _invoke_function_over_columns("var_pop", col) + return _invoke_function_over_columns("ln", col) @_try_remote_functions -def regr_avgx(y: "ColumnOrName", x: "ColumnOrName") -> Column: - """ - Aggregate function: returns the average of the independent variable for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. +def log2(col: "ColumnOrName") -> Column: + """Returns the base-2 logarithm of the argument. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + a column to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - the average of the independent variable for non-null pairs in a group. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + logariphm of given value. + Returns a column that evaluates to a double. Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | 2.75| 2.75| - +---------------+------+ - - Example 2: All pairs' x values are null + >>> from pyspark.sql import functions as sf + >>> spark.range(10).select("*", sf.log2('id')).show() + +---+------------------+ + | id| LOG2(id)| + +---+------------------+ + | 0| NULL| + | 1| 0.0| + | 2| 1.0| + | 3| 1.584962500721...| + | 4| 2.0| + | 5| 2.321928094887...| + | 6| 2.584962500721...| + | 7| 2.807354922057...| + | 8| 3.0| + | 9|3.1699250014423...| + +---+------------------+ + """ + return _invoke_function_over_columns("log2", col) - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | NULL| NULL| - +---------------+------+ - Example 3: All pairs' y values are null +@_try_remote_functions +def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column: + """ + Convert a number in a string column from one base to another. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | NULL| 1.0| - +---------------+------+ + .. versionadded:: 1.5.0 - Example 4: Some pairs' x values are null + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | 3.0| 3.0| - +---------------+------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column to convert base for. + A column that evaluates to a string. + fromBase: int + from base number. + A column that evaluates to an integer. + toBase: int + to base number. + A column that evaluates to an integer. - Example 5: Some pairs' x or y values are null + Returns + ------- + :class:`~pyspark.sql.Column` + logariphm of given value. + Returns a column that evaluates to a string. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | 3.0| 3.0| - +---------------+------+ + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("010101",), ( "101",), ("001",)], ['n']) + >>> df.select("*", sf.conv(df.n, 2, 16)).show() + +------+--------------+ + | n|conv(n, 2, 16)| + +------+--------------+ + |010101| 15| + | 101| 5| + | 001| 1| + +------+--------------+ """ - return _invoke_function_over_columns("regr_avgx", y, x) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "conv", _to_java_column(col), _enum_to_value(fromBase), _enum_to_value(toBase) + ) @_try_remote_functions -def regr_avgy(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def factorial(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the average of the dependent variable for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Computes the factorial of the given value. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + a column to calculate factorial for. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - the average of the dependent variable for non-null pairs in a group. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + factorial of given value. + Returns a column that evaluates to a long. Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------+ - |regr_avgy(y, x)|avg(y)| - +---------------+------+ - | 1.75| 1.75| - +---------------+------+ - - Example 2: All pairs' x values are null + >>> from pyspark.sql import functions as sf + >>> spark.range(10).select("*", sf.factorial('id')).show() + +---+-------------+ + | id|factorial(id)| + +---+-------------+ + | 0| 1| + | 1| 1| + | 2| 2| + | 3| 6| + | 4| 24| + | 5| 120| + | 6| 720| + | 7| 5040| + | 8| 40320| + | 9| 362880| + +---+-------------+ + """ + return _invoke_function_over_columns("factorial", col) - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------+ - |regr_avgy(y, x)|avg(y)| - +---------------+------+ - | NULL| 1.0| - +---------------+------+ - Example 3: All pairs' y values are null +@_try_remote_functions +def bin(col: "ColumnOrName") -> Column: + """Returns the string representation of the binary value of the given column. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------+ - |regr_avgy(y, x)|avg(y)| - +---------------+------+ - | NULL| NULL| - +---------------+------+ + .. versionadded:: 1.5.0 - Example 4: Some pairs' x values are null + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +------------------+------+ - | regr_avgy(y, x)|avg(y)| - +------------------+------+ - |1.6666666666666...| 1.75| - +------------------+------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a long. - Example 5: Some pairs' x or y values are null + Returns + ------- + :class:`~pyspark.sql.Column` + binary representation of given value as string. + Returns a column that evaluates to a string. + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------------------+ - |regr_avgy(y, x)| avg(y)| - +---------------+------------------+ - | 1.5|1.6666666666666...| - +---------------+------------------+ + >>> spark.range(10).select("*", sf.bin("id")).show() + +---+-------+ + | id|bin(id)| + +---+-------+ + | 0| 0| + | 1| 1| + | 2| 10| + | 3| 11| + | 4| 100| + | 5| 101| + | 6| 110| + | 7| 111| + | 8| 1000| + | 9| 1001| + +---+-------+ """ - return _invoke_function_over_columns("regr_avgy", y, x) + return _invoke_function_over_columns("bin", col) @_try_remote_functions -def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: - """ - Aggregate function: returns the number of non-null number pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. +def hex(col: "ColumnOrName") -> Column: + """Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`, + :class:`pyspark.sql.types.BinaryType`, :class:`pyspark.sql.types.IntegerType` or + :class:`pyspark.sql.types.LongType`. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a long, binary, or string. See Also -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + :meth:`pyspark.sql.functions.unhex` Returns ------- :class:`~pyspark.sql.Column` - the number of non-null number pairs in a group. + hexadecimal representation of given value as string. + Returns a column that evaluates to a string. Examples -------- - Example 1: All pairs are non-null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 4| 4| - +----------------+--------+ + >>> df = spark.createDataFrame([('ABC', 3)], ['a', 'b']) + >>> df.select('*', sf.hex('a'), sf.hex(df.b)).show() + +---+---+------+------+ + | a| b|hex(a)|hex(b)| + +---+---+------+------+ + |ABC| 3|414243| 3| + +---+---+------+------+ + """ + return _invoke_function_over_columns("hex", col) - Example 2: All pairs' x values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 0| 1| - +----------------+--------+ +@_try_remote_functions +def unhex(col: "ColumnOrName") -> Column: + """Inverse of hex. Interprets each pair of characters as a hexadecimal number + and converts to the byte representation of number. - Example 3: All pairs' y values are null + .. versionadded:: 1.5.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 0| 1| - +----------------+--------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Example 4: Some pairs' x values are null + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 3| 4| - +----------------+--------+ + See Also + -------- + :meth:`pyspark.sql.functions.hex` - Example 5: Some pairs' x or y values are null + Returns + ------- + :class:`~pyspark.sql.Column` + byte representation of the given hexadecimal value. + Returns a column that evaluates to a binary. + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 2| 4| - +----------------+--------+ + >>> df = spark.createDataFrame([('414243',)], ['a']) + >>> df.select('*', sf.unhex('a')).show() + +------+----------+ + | a| unhex(a)| + +------+----------+ + |414243|[41 42 43]| + +------+----------+ """ - return _invoke_function_over_columns("regr_count", y, x) + return _invoke_function_over_columns("unhex", col) @_try_remote_functions -def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: - """ - Aggregate function: returns the intercept of the univariate linear regression line - for non-null pairs in a group, where `y` is the dependent variable and - `x` is the independent variable. +def uniform( + min: Union[Column, int, float], + max: Union[Column, int, float], + seed: Optional[Union[Column, int]] = None, +) -> Column: + """Returns a random value with independent and identically distributed (i.i.d.) values with the + specified range of numbers. The random seed is optional. The provided numbers specifying the + minimum and maximum values of the range must be constant. If both of these numbers are integers, + then the result will also be an integer. Otherwise if one or both of these are floating-point + numbers, then the result will also be a floating-point number. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + min : :class:`~pyspark.sql.Column`, int, or float + Minimum value in the range. + A column that evaluates to a numeric. Must be a constant. + max : :class:`~pyspark.sql.Column`, int, or float + Maximum value in the range. + A column that evaluates to a numeric. Must be a constant. + seed : :class:`~pyspark.sql.Column` or int + Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - the intercept of the univariate linear regression line for non-null pairs in a group. + The generated random number within the specified range. + Returns a column of the same type as the input. See Also -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + :meth:`pyspark.sql.functions.rand` + :meth:`pyspark.sql.functions.randn` Examples -------- - Example 1: All pairs are non-null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | 0.0| - +--------------------+ + >>> spark.range(0, 10, 1, 1).select(sf.uniform(5, 105, 3)).show() + +------------------+ + |uniform(5, 105, 3)| + +------------------+ + | 30| + | 71| + | 99| + | 77| + | 16| + | 25| + | 89| + | 80| + | 51| + | 83| + +------------------+ + """ + min = _enum_to_value(min) + min = lit(min) + max = _enum_to_value(max) + max = lit(max) + if seed is None: + return _invoke_function_over_columns("uniform", min, max) + else: + seed = _enum_to_value(seed) + seed = lit(seed) + return _invoke_function_over_columns("uniform", min, max, seed) - Example 2: All pairs' x values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | NULL| - +--------------------+ +# ---------------------- String Functions ---------------------- - Example 3: All pairs' y values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | NULL| - +--------------------+ +@_try_remote_functions +def upper(col: "ColumnOrName") -> Column: + """ + Converts a string expression to upper case. - Example 4: Some pairs' x values are null + .. versionadded:: 1.5.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | 0.0| - +--------------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Example 5: Some pairs' x or y values are null + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | 0.0| - +--------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + upper case values. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.lower` + :meth:`pyspark.sql.functions.ucase` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.upper("value")).show() + +----------+------------+ + | value|upper(value)| + +----------+------------+ + | Spark| SPARK| + | PySpark| PYSPARK| + |Pandas API| PANDAS API| + +----------+------------+ """ - return _invoke_function_over_columns("regr_intercept", y, x) + return _invoke_function_over_columns("upper", col) @_try_remote_functions -def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def lower(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the coefficient of determination for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Converts a string expression to lower case. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the coefficient of determination for non-null pairs in a group. + lower case values. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + :meth:`pyspark.sql.functions.upper` + :meth:`pyspark.sql.functions.lcase` Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | 1.0| - +-------------+ - - Example 2: All pairs' x values are null + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.lower("value")).show() + +----------+------------+ + | value|lower(value)| + +----------+------------+ + | Spark| spark| + | PySpark| pyspark| + |Pandas API| pandas api| + +----------+------------+ + """ + return _invoke_function_over_columns("lower", col) - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | NULL| - +-------------+ - Example 3: All pairs' y values are null +@_try_remote_functions +def ascii(col: "ColumnOrName") -> Column: + """ + Computes the numeric value of the first character of the string column. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | NULL| - +-------------+ + .. versionadded:: 1.5.0 - Example 4: Some pairs' x values are null + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | 1.0| - +-------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. - Example 5: Some pairs' x or y values are null + Returns + ------- + :class:`~pyspark.sql.Column` + numeric value. + Returns a column that evaluates to an integer. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | 1.0| - +-------------+ + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.ascii("value")).show() + +----------+------------+ + | value|ascii(value)| + +----------+------------+ + | Spark| 83| + | PySpark| 80| + |Pandas API| 80| + +----------+------------+ """ - return _invoke_function_over_columns("regr_r2", y, x) + return _invoke_function_over_columns("ascii", col) @_try_remote_functions -def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def base64(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the slope of the linear regression line for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Computes the BASE64 encoding of a binary column and returns it as a string column. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - the slope of the linear regression line for non-null pairs in a group. + BASE64 encoding of string value. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + :meth:`pyspark.sql.functions.unbase64` + :meth:`pyspark.sql.functions.to_base32` Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | 1.0| - +----------------+ - - Example 2: All pairs' x values are null + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.base64("value")).show() + +----------+----------------+ + | value| base64(value)| + +----------+----------------+ + | Spark| U3Bhcms=| + | PySpark| UHlTcGFyaw==| + |Pandas API|UGFuZGFzIEFQSQ==| + +----------+----------------+ + """ + return _invoke_function_over_columns("base64", col) - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | NULL| - +----------------+ - Example 3: All pairs' y values are null +@_try_remote_functions +def to_base32(col: "ColumnOrName") -> Column: + """ + Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a + string column. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | NULL| - +----------------+ + .. versionadded:: 4.3.0 - Example 4: Some pairs' x values are null + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a binary. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | 1.0| - +----------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + BASE32 encoding of the binary value. + Returns a column that evaluates to a string. - Example 5: Some pairs' x or y values are null + See Also + -------- + :meth:`pyspark.sql.functions.from_base32` + :meth:`pyspark.sql.functions.base64` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | 1.0| - +----------------+ + >>> df = spark.createDataFrame([(b"foobar",)], ["value"]) + >>> df.select(sf.to_base32("value").alias("r")).collect() + [Row(r='MZXW6YTBOI======')] """ - return _invoke_function_over_columns("regr_slope", y, x) + return _invoke_function_over_columns("to_base32", col) @_try_remote_functions -def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def unbase64(col: "ColumnOrName") -> Column: """ - Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Decodes a BASE64 encoded string column and returns it as a binary column. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group. + decoded binary value. + Returns a column that evaluates to a binary. See Also -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + :meth:`pyspark.sql.functions.base64` + :meth:`pyspark.sql.functions.from_base32` Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | 5.0| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["U3Bhcms=", "UHlTcGFyaw==", "UGFuZGFzIEFQSQ=="], "STRING") + >>> df.select("*", sf.unbase64("value")).show(truncate=False) + +----------------+-------------------------------+ + |value |unbase64(value) | + +----------------+-------------------------------+ + |U3Bhcms= |[53 70 61 72 6B] | + |UHlTcGFyaw== |[50 79 53 70 61 72 6B] | + |UGFuZGFzIEFQSQ==|[50 61 6E 64 61 73 20 41 50 49]| + +----------------+-------------------------------+ + """ + return _invoke_function_over_columns("unbase64", col) - Example 2: All pairs' x values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | NULL| - +--------------+ +@_try_remote_functions +def from_base32(col: "ColumnOrName") -> Column: + """ + Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary + column. - Example 3: All pairs' y values are null + .. versionadded:: 4.3.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | NULL| - +--------------+ - - Example 4: Some pairs' x values are null + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +-----------------+ - | regr_sxx(y, x)| - +-----------------+ - |4.666666666666...| - +-----------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + decoded binary value. + Returns a column that evaluates to a binary. - Example 5: Some pairs' x or y values are null + See Also + -------- + :meth:`pyspark.sql.functions.to_base32` + :meth:`pyspark.sql.functions.unbase64` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | 4.5| - +--------------+ + >>> df = spark.createDataFrame([("MZXW6YTBOI======",)], ["value"]) + >>> df.select(sf.from_base32("value").alias("r")).collect() + [Row(r=b'foobar')] """ - return _invoke_function_over_columns("regr_sxx", y, x) + return _invoke_function_over_columns("from_base32", col) @_try_remote_functions -def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: """ - Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Trim the spaces from left end for the specified string value. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or column name, optional + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_syy` + .. versionadded:: 4.0.0 Returns ------- :class:`~pyspark.sql.Column` - REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group. + left trimmed values. + Returns a column that evaluates to a string. - Examples + See Also -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | 5.0| - +--------------+ - - Example 2: All pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | NULL| - +--------------+ + :meth:`pyspark.sql.functions.trim` + :meth:`pyspark.sql.functions.rtrim` - Example 3: All pairs' y values are null + Examples + -------- + Example 1: Trim the spaces - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | NULL| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select("*", sf.ltrim("value")).show() + +--------+------------+ + | value|ltrim(value)| + +--------+------------+ + | Spark| Spark| + | Spark | Spark | + | Spark| Spark| + +--------+------------+ - Example 4: Some pairs' x values are null + Example 2: Trim specified characters - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +-----------------+ - | regr_sxy(y, x)| - +-----------------+ - |4.666666666666...| - +-----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") + >>> df.select("*", sf.ltrim("value", sf.lit("*"))).show() + +--------+--------------------------+ + | value|TRIM(LEADING * FROM value)| + +--------+--------------------------+ + |***Spark| Spark| + | Spark**| Spark**| + | *Spark| Spark| + +--------+--------------------------+ - Example 5: Some pairs' x or y values are null + Example 3: Trim a column containing different characters - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | 4.5| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) + >>> df.select("*", sf.ltrim("value", "t")).show() + +--------+---+--------------------------+ + | value| t|TRIM(LEADING t FROM value)| + +--------+---+--------------------------+ + |**Spark*| *| Spark*| + |==Spark=| =| Spark=| + +--------+---+--------------------------+ """ - return _invoke_function_over_columns("regr_sxy", y, x) + if trim is not None: + return _invoke_function_over_columns("ltrim", col, trim) + else: + return _invoke_function_over_columns("ltrim", col) @_try_remote_functions -def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: """ - Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Trim the spaces from right end for the specified string value. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or column name, optional + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. + + .. versionadded:: 4.0.0 Returns ------- :class:`~pyspark.sql.Column` - REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group. + right trimmed values. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.trim` + :meth:`pyspark.sql.functions.ltrim` Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | 5.0| - +--------------+ - - Example 2: All pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | NULL| - +--------------+ - - Example 3: All pairs' y values are null + Example 1: Trim the spaces - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | NULL| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select("*", sf.rtrim("value")).show() + +--------+------------+ + | value|rtrim(value)| + +--------+------------+ + | Spark| Spark| + | Spark | Spark| + | Spark| Spark| + +--------+------------+ - Example 4: Some pairs' x values are null + Example 2: Trim specified characters - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +-----------------+ - | regr_syy(y, x)| - +-----------------+ - |4.666666666666...| - +-----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") + >>> df.select("*", sf.rtrim("value", sf.lit("*"))).show() + +--------+---------------------------+ + | value|TRIM(TRAILING * FROM value)| + +--------+---------------------------+ + |***Spark| ***Spark| + | Spark**| Spark| + | *Spark| *Spark| + +--------+---------------------------+ - Example 5: Some pairs' x or y values are null + Example 3: Trim a column containing different characters - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | 4.5| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) + >>> df.select("*", sf.rtrim("value", "t")).show() + +--------+---+---------------------------+ + | value| t|TRIM(TRAILING t FROM value)| + +--------+---+---------------------------+ + |**Spark*| *| **Spark| + |==Spark=| =| ==Spark| + +--------+---+---------------------------+ """ - return _invoke_function_over_columns("regr_syy", y, x) + if trim is not None: + return _invoke_function_over_columns("rtrim", col, trim) + else: + return _invoke_function_over_columns("rtrim", col) @_try_remote_functions -def every(col: "ColumnOrName") -> Column: +def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: """ - Aggregate function: returns true if all values of `col` are true. + Trim the spaces from both ends for the specified string column. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column to check if all values are true. - A column that evaluates to a boolean. + target column to work on. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or column name, optional + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. - See Also - -------- - :meth:`pyspark.sql.functions.some` + .. versionadded:: 4.0.0 Returns ------- :class:`~pyspark.sql.Column` - true if all values of `col` are true, false otherwise. + trimmed values from both sides. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.ltrim` + :meth:`pyspark.sql.functions.rtrim` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [True], [True]], ["flag"] - ... ).select(sf.every("flag")).show() - +-----------+ - |every(flag)| - +-----------+ - | true| - +-----------+ + Example 1: Trim the spaces - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [False], [True]], ["flag"] - ... ).select(sf.every("flag")).show() - +-----------+ - |every(flag)| - +-----------+ - | false| - +-----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select("*", sf.trim("value")).show() + +--------+-----------+ + | value|trim(value)| + +--------+-----------+ + | Spark| Spark| + | Spark | Spark| + | Spark| Spark| + +--------+-----------+ - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[False], [False], [False]], ["flag"] - ... ).select(sf.every("flag")).show() - +-----------+ - |every(flag)| - +-----------+ - | false| - +-----------+ + Example 2: Trim specified characters + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") + >>> df.select("*", sf.trim("value", sf.lit("*"))).show() + +--------+-----------------------+ + | value|TRIM(BOTH * FROM value)| + +--------+-----------------------+ + |***Spark| Spark| + | Spark**| Spark| + | *Spark| Spark| + +--------+-----------------------+ + + Example 3: Trim a column containing different characters + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) + >>> df.select("*", sf.trim("value", "t")).show() + +--------+---+-----------------------+ + | value| t|TRIM(BOTH t FROM value)| + +--------+---+-----------------------+ + |**Spark*| *| Spark| + |==Spark=| =| Spark| + +--------+---+-----------------------+ """ - return _invoke_function_over_columns("every", col) + if trim is not None: + return _invoke_function_over_columns("trim", col, trim) + else: + return _invoke_function_over_columns("trim", col) @_try_remote_functions -def bool_and(col: "ColumnOrName") -> Column: +def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: """ - Aggregate function: returns true if all values of `col` are true. + Concatenates multiple input string columns together into a single string column, + using the given separator. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to check if all values are true. - A column that evaluates to a boolean. + sep : literal string + words separator. + A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or column name + list of columns to work on. + Each a column that evaluates to a string or an array of strings. Returns ------- :class:`~pyspark.sql.Column` - true if all values of `col` are true, false otherwise. + string of concatenated words. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.bool_or` + :meth:`pyspark.sql.functions.concat` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) - >>> df.select(sf.bool_and("flag")).show() - +--------------+ - |bool_and(flag)| - +--------------+ - | true| - +--------------+ - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) - >>> df.select(sf.bool_and("flag")).show() - +--------------+ - |bool_and(flag)| - +--------------+ - | false| - +--------------+ - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) - >>> df.select(sf.bool_and("flag")).show() - +--------------+ - |bool_and(flag)| - +--------------+ - | false| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("abcd", "123")], ["s", "d"]) + >>> df.select("*", sf.concat_ws("-", df.s, "d", sf.lit("xyz"))).show() + +----+---+-----------------------+ + | s| d|concat_ws(-, s, d, xyz)| + +----+---+-----------------------+ + |abcd|123| abcd-123-xyz| + +----+---+-----------------------+ """ - return _invoke_function_over_columns("bool_and", col) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function("concat_ws", _enum_to_value(sep), _to_seq(sc, cols, _to_java_column)) @_try_remote_functions -def some(col: "ColumnOrName") -> Column: +def decode(col: "ColumnOrName", charset: str) -> Column: """ - Aggregate function: returns true if at least one value of `col` is true. + Computes the first argument into a string from a binary using the provided character set + (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column to check if at least one value is true. - A column that evaluates to a boolean. + target column to work on. + charset : literal string + charset to use to decode to. Returns ------- :class:`~pyspark.sql.Column` - true if at least one value of `col` is true, false otherwise. + the column for computed results. See Also -------- - :meth:`pyspark.sql.functions.every` + :meth:`pyspark.sql.functions.encode` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [True], [True]], ["flag"] - ... ).select(sf.some("flag")).show() - +----------+ - |some(flag)| - +----------+ - | true| - +----------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [False], [True]], ["flag"] - ... ).select(sf.some("flag")).show() - +----------+ - |some(flag)| - +----------+ - | true| - +----------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[False], [False], [False]], ["flag"] - ... ).select(sf.some("flag")).show() - +----------+ - |some(flag)| - +----------+ - | false| - +----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b"\x61\x62\x63\x64",)], ["a"]) + >>> df.select("*", sf.decode("a", "UTF-8")).show() + +-------------+----------------+ + | a|decode(a, UTF-8)| + +-------------+----------------+ + |[61 62 63 64]| abcd| + +-------------+----------------+ """ - return _invoke_function_over_columns("some", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("decode", _to_java_column(col), _enum_to_value(charset)) @_try_remote_functions -def bool_or(col: "ColumnOrName") -> Column: +def encode(col: "ColumnOrName", charset: str) -> Column: """ - Aggregate function: returns true if at least one value of `col` is true. + Computes the first argument into a binary from a string using the provided character set + (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column to check if at least one value is true. - A column that evaluates to a boolean. + target column to work on. + A column that evaluates to a string. + charset : literal string + charset to use to encode. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - true if at least one value of `col` is true, false otherwise. + the column for computed results. + Returns a column that evaluates to a binary. See Also -------- - :meth:`pyspark.sql.functions.bool_and` + :meth:`pyspark.sql.functions.decode` Examples -------- - >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) - >>> df.select(bool_or("flag")).show() - +-------------+ - |bool_or(flag)| - +-------------+ - | true| - +-------------+ - >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) - >>> df.select(bool_or("flag")).show() - +-------------+ - |bool_or(flag)| - +-------------+ - | true| - +-------------+ - >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) - >>> df.select(bool_or("flag")).show() - +-------------+ - |bool_or(flag)| - +-------------+ - | false| - +-------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("abcd",)], ["c"]) + >>> df.select("*", sf.encode("c", "UTF-8")).show() + +----+----------------+ + | c|encode(c, UTF-8)| + +----+----------------+ + |abcd| [61 62 63 64]| + +----+----------------+ """ - return _invoke_function_over_columns("bool_or", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("encode", _to_java_column(col), _enum_to_value(charset)) @_try_remote_functions -def bit_and(col: "ColumnOrName") -> Column: +def is_valid_utf8(str: "ColumnOrName") -> Column: """ - Aggregate function: returns the bitwise AND of all non-null input values, or null if none. + Returns true if the input is a valid UTF-8 string, otherwise returns false. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the bitwise AND of all non-null input values, or null if none. + whether the input string is a valid UTF-8 string. + Returns a column that evaluates to a boolean. See Also -------- - :meth:`pyspark.sql.functions.bit_or` - :meth:`pyspark.sql.functions.bit_xor` + :meth:`pyspark.sql.functions.make_valid_utf8` + :meth:`pyspark.sql.functions.validate_utf8` + :meth:`pyspark.sql.functions.try_validate_utf8` Examples -------- - Example 1: Bitwise AND with all non-null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | 0| - +----------+ - - Example 2: Bitwise AND with null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | 0| - +----------+ - - Example 3: Bitwise AND with all null values - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([StructField("c", IntegerType(), True)]) - >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | NULL| - +----------+ - - Example 4: Bitwise AND with single input value - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[5]], ["c"]) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | 5| - +----------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.is_valid_utf8(sf.lit("SparkSQL"))).show() + +-----------------------+ + |is_valid_utf8(SparkSQL)| + +-----------------------+ + | true| + +-----------------------+ """ - return _invoke_function_over_columns("bit_and", col) + return _invoke_function_over_columns("is_valid_utf8", str) @_try_remote_functions -def bit_or(col: "ColumnOrName") -> Column: +def make_valid_utf8(str: "ColumnOrName") -> Column: """ - Aggregate function: returns the bitwise OR of all non-null input values, or null if none. + Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the + Unicode replacement character (U+FFFD). - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the bitwise OR of all non-null input values, or null if none. + the valid UTF-8 version of the given input string. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.bit_and` - :meth:`pyspark.sql.functions.bit_xor` + :meth:`pyspark.sql.functions.is_valid_utf8` + :meth:`pyspark.sql.functions.validate_utf8` + :meth:`pyspark.sql.functions.try_validate_utf8` Examples -------- - Example 1: Bitwise OR with all non-null values + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.make_valid_utf8(sf.lit("SparkSQL"))).show() + +-------------------------+ + |make_valid_utf8(SparkSQL)| + +-------------------------+ + | SparkSQL| + +-------------------------+ + """ + return _invoke_function_over_columns("make_valid_utf8", str) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | 3| - +---------+ - Example 2: Bitwise OR with some null values +@_try_remote_functions +def validate_utf8(str: "ColumnOrName") -> Column: + """ + Returns the input value if it corresponds to a valid UTF-8 string, or emits an error otherwise. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | 3| - +---------+ + .. versionadded:: 4.0.0 - Example 3: Bitwise OR with all null values + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([StructField("c", IntegerType(), True)]) - >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | NULL| - +---------+ + Returns + ------- + :class:`~pyspark.sql.Column` + the input string if it is a valid UTF-8 string, error otherwise. + Returns a column that evaluates to a string. - Example 4: Bitwise OR with single input value + See Also + -------- + :meth:`pyspark.sql.functions.is_valid_utf8` + :meth:`pyspark.sql.functions.make_valid_utf8` + :meth:`pyspark.sql.functions.try_validate_utf8` - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[5]], ["c"]) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | 5| - +---------+ + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.validate_utf8(sf.lit("SparkSQL"))).show() + +-----------------------+ + |validate_utf8(SparkSQL)| + +-----------------------+ + | SparkSQL| + +-----------------------+ """ - return _invoke_function_over_columns("bit_or", col) + return _invoke_function_over_columns("validate_utf8", str) @_try_remote_functions -def bit_xor(col: "ColumnOrName") -> Column: +def try_validate_utf8(str: "ColumnOrName") -> Column: """ - Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. + Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the bitwise XOR of all non-null input values, or null if none. + the input string if it is a valid UTF-8 string, null otherwise. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.bit_and` - :meth:`pyspark.sql.functions.bit_or` + :meth:`pyspark.sql.functions.is_valid_utf8` + :meth:`pyspark.sql.functions.make_valid_utf8` + :meth:`pyspark.sql.functions.validate_utf8` Examples -------- - Example 1: Bitwise XOR with all non-null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | 2| - +----------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.try_validate_utf8(sf.lit("SparkSQL"))).show() + +---------------------------+ + |try_validate_utf8(SparkSQL)| + +---------------------------+ + | SparkSQL| + +---------------------------+ + """ + return _invoke_function_over_columns("try_validate_utf8", str) - Example 2: Bitwise XOR with some null values - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | 3| - +----------+ +@_try_remote_functions +def normalize(str: "ColumnOrName", form: Optional["ColumnOrName"] = None) -> Column: + """ + Returns the Unicode normalization of ``str`` using the given normalization ``form``, as + defined by Unicode Standard Annex #15. Normalization is backed by Spark's bundled ICU4J + library rather than the JVM's own Unicode data, so results are stable across JVM vendors + and versions. - Example 3: Bitwise XOR with all null values + .. versionadded:: 4.4.0 - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([StructField("c", IntegerType(), True)]) - >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | NULL| - +----------+ + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or column name + the input string to normalize. + form : :class:`~pyspark.sql.Column` or column name, optional + the normalization form, one of 'NFC', 'NFD', 'NFKC', 'NFKD' (case-insensitive). + If omitted, 'NFC' is used. - Example 4: Bitwise XOR with single input value + Returns + ------- + :class:`~pyspark.sql.Column` + the normalized string. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[5]], ["c"]) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | 5| - +----------+ + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("\ufb01",)], ["s"]) + >>> df.select(sf.normalize(df.s, sf.lit("NFKC"))).show() + +------------------+ + |normalize(s, NFKC)| + +------------------+ + | fi| + +------------------+ """ - return _invoke_function_over_columns("bit_xor", col) + if form is None: + return _invoke_function_over_columns("normalize", str) + else: + return _invoke_function_over_columns("normalize", str, form) @_try_remote_functions -def skewness(col: "ColumnOrName") -> Column: +def format_number(col: "ColumnOrName", d: int) -> Column: """ - Aggregate function: returns the skewness of the values in a group. + Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places + with HALF_EVEN round mode, and returns the result as a string. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -6021,1363 +5920,1619 @@ def skewness(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + the column name of the numeric value to be formatted. A column that evaluates to a numeric. - - See Also - -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.kurtosis` + d : int + the N decimal places. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - skewness of given column. + the column of formatted results. + Returns a column that evaluates to a string. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.skewness(df.c)).show() - +------------------+ - | skewness(c)| - +------------------+ - |0.7071067811865...| - +------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(5,)], ["a"]) + >>> df.select("*", sf.format_number("a", 4), sf.format_number(df.a, 6)).show() + +---+-------------------+-------------------+ + | a|format_number(a, 4)|format_number(a, 6)| + +---+-------------------+-------------------+ + | 5| 5.0000| 5.000000| + +---+-------------------+-------------------+ """ - return _invoke_function_over_columns("skewness", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("format_number", _to_java_column(col), _enum_to_value(d)) @_try_remote_functions -def kurtosis(col: "ColumnOrName") -> Column: +def format_string(format: str, *cols: "ColumnOrName") -> Column: """ - Aggregate function: returns the kurtosis of the values in a group. + Formats the arguments in printf-style and returns the result as a string column. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + format : literal string + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or column name + column names or :class:`~pyspark.sql.Column`\\s to be used in formatting + Each a column of any type. Returns ------- :class:`~pyspark.sql.Column` - kurtosis of given column. + the column of formatted results. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.skewness` + :meth:`pyspark.sql.functions.printf` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.kurtosis(df.c)).show() - +-----------+ - |kurtosis(c)| - +-----------+ - | -1.5| - +-----------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(5, "hello")], ["a", "b"]) + >>> df.select("*", sf.format_string('%d %s', "a", df.b)).show() + +---+-----+--------------------------+ + | a| b|format_string(%d %s, a, b)| + +---+-----+--------------------------+ + | 5|hello| 5 hello| + +---+-----+--------------------------+ """ - return _invoke_function_over_columns("kurtosis", col) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function( + "format_string", _enum_to_value(format), _to_seq(sc, cols, _to_java_column) + ) @_try_remote_functions -def collect_list(col: "ColumnOrName") -> Column: +def instr( + str: "ColumnOrName", + substr: Union[Column, str], + start: Optional[Union[Column, int]] = None, + occurrence: Optional[Union[Column, int]] = None, +) -> Column: """ - Aggregate function: Collects the values from a column into a list, - maintaining duplicates, and returns this list of objects. + Locate the position of the specified occurrence of substr column in the given string. + Returns null if either of the arguments are null. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.3.0 + Supports optional `start` and `occurrence` parameters. + + Notes + ----- + The position is not zero based, but 1 based index. Returns 0 if substr + could not be found in str. + Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the function is computed. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + substr : :class:`~pyspark.sql.Column` or literal string + substring to look for. + A column that evaluates to a string. - See Also - -------- - :meth:`pyspark.sql.functions.array_agg` - :meth:`pyspark.sql.functions.collect_set` + .. versionchanged:: 4.0.0 + `substr` now accepts column. + start : int or :class:`~pyspark.sql.Column`, optional + Starting position (1-based, can be negative for backward search). + If not specified, defaults to 1. + A column that evaluates to an integer. + occurrence : int or :class:`~pyspark.sql.Column`, optional + Which occurrence to locate (must be > 0). Defaults to 1. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing a list of collected values, with duplicate values included. + location of the substring as integer. + Returns a column that evaluates to an integer. - Notes - ----- - The function is non-deterministic as the order of collected results depends - on the order of the rows, which possibly becomes non-deterministic after shuffle operations. + See Also + -------- + :meth:`pyspark.sql.functions.locate` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.functions.substring_index` Examples -------- - Example 1: Collect values from a DataFrame and sort the result in ascending order + Example 1: Using a literal string as the 'substring' >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) - >>> df.select(sf.sort_array(sf.collect_list('value')).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [1, 2, 2]| - +-----------+ + >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr(df.s, "b")).show() + +----+-----------+ + | s|instr(s, b)| + +----+-----------+ + |abcd| 2| + | xyz| 0| + +----+-----------+ - Example 2: Collect values from a DataFrame and sort the result in descending order + Example 2: Using a Column 'substring' >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) - >>> df.select(sf.sort_array(sf.collect_list('age'), asc=False).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [5, 5, 2]| - +-----------+ + >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr("s", sf.lit("abc").substr(0, 2))).show() + +----+---------------------------+ + | s|instr(s, substr(abc, 0, 2))| + +----+---------------------------+ + |abcd| 1| + | xyz| 0| + +----+---------------------------+ - Example 3: Collect values from a DataFrame with multiple columns and sort the result + Example 3: Using start and occurrence parameters >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) - >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_list('id')).alias('sorted_list')) - >>> df.orderBy(sf.desc("name")).show() - +----+-----------+ - |name|sorted_list| - +----+-----------+ - |John| [1, 2]| - | Ana| [3]| - +----+-----------+ + >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr("s", "b", 1, 2)).show() + +-----+-----------------+ + | s|instr(s, b, 1, 2)| + +-----+-----------------+ + |aabcd| 0| + | xyz| 0| + +-----+-----------------+ + + Example 4: Using start parameter + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr("s", "a", 2)).show() + +-----+-----------------+ + | s|instr(s, a, 2, 1)| + +-----+-----------------+ + |aabcd| 2| + | xyz| 0| + +-----+-----------------+ """ - return _invoke_function_over_columns("collect_list", col) + if start is None and occurrence is None: + return _invoke_function_over_columns("instr", str, lit(substr)) + elif start is not None and occurrence is None: + start = lit(start) + return _invoke_function_over_columns("instr", str, lit(substr), start) + else: + start = lit(start) if start is not None else lit(1) + occurrence = lit(occurrence) + return _invoke_function_over_columns("instr", str, lit(substr), start, occurrence) @_try_remote_functions -def array_agg(col: "ColumnOrName") -> Column: +def overlay( + src: "ColumnOrName", + replace: "ColumnOrName", + pos: Union["ColumnOrName", int], + len: Union["ColumnOrName", int] = -1, +) -> Column: """ - Aggregate function: returns a list of objects with duplicates. + Overlay the specified portion of `src` with `replace`, + starting from byte position `pos` of `src` and proceeding for `len` bytes. - .. versionadded:: 3.5.0 + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + src : :class:`~pyspark.sql.Column` or column name + the string that will be replaced. + A column that evaluates to a string or binary. + replace : :class:`~pyspark.sql.Column` or column name + the substitution string. + A column that evaluates to a string or binary. + pos : :class:`~pyspark.sql.Column` or column name or int + the starting position in src. + A column that evaluates to an integer. + len : :class:`~pyspark.sql.Column` or column name or int, optional + the number of bytes to replace in src + string by 'replace' defaults to -1, which represents the length of the 'replace' string. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - list of objects with duplicates. - - See Also - -------- - :meth:`pyspark.sql.functions.collect_list` - :meth:`pyspark.sql.functions.collect_set` + string with replaced values. + Returns a column of the same type as the input. Examples -------- - Example 1: Using array_agg function on an int column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [1, 1, 2]| - +-----------+ - - Example 2: Using array_agg function on a string column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([["apple"],["apple"],["banana"]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show(truncate=False) - +----------------------+ - |sorted_list | - +----------------------+ - |[apple, apple, banana]| - +----------------------+ + >>> df = spark.createDataFrame([("SPARK_SQL", "CORE")], ("x", "y")) + >>> df.select("*", sf.overlay("x", df.y, 7)).show() + +---------+----+--------------------+ + | x| y|overlay(x, y, 7, -1)| + +---------+----+--------------------+ + |SPARK_SQL|CORE| SPARK_CORE| + +---------+----+--------------------+ - Example 3: Using array_agg function on a column with null values + >>> df.select("*", sf.overlay("x", df.y, 7, 0)).show() + +---------+----+-------------------+ + | x| y|overlay(x, y, 7, 0)| + +---------+----+-------------------+ + |SPARK_SQL|CORE| SPARK_CORESQL| + +---------+----+-------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [1, 2]| - +-----------+ + >>> df.select("*", sf.overlay("x", "y", 7, 2)).show() + +---------+----+-------------------+ + | x| y|overlay(x, y, 7, 2)| + +---------+----+-------------------+ + |SPARK_SQL|CORE| SPARK_COREL| + +---------+----+-------------------+ + """ + pos = _enum_to_value(pos) + if not isinstance(pos, (int, str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column, int or str", + "arg_name": "pos", + "arg_type": type(pos).__name__, + }, + ) + len = _enum_to_value(len) + if len is not None and not isinstance(len, (int, str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column, int or str", + "arg_name": "len", + "arg_type": type(len).__name__, + }, + ) - Example 4: Using array_agg function on a column with different data types + if isinstance(pos, int): + pos = lit(pos) + if isinstance(len, int): + len = lit(len) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],["apple"],[2]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() - +-------------+ - | sorted_list| - +-------------+ - |[1, 2, apple]| - +-------------+ - """ - return _invoke_function_over_columns("array_agg", col) + return _invoke_function_over_columns("overlay", src, replace, pos, len) @_try_remote_functions -def collect_set(col: "ColumnOrName") -> Column: +def sentences( + string: "ColumnOrName", + language: Optional["ColumnOrName"] = None, + country: Optional["ColumnOrName"] = None, +) -> Column: """ - Aggregate function: Collects the values from a column into a set, - eliminating duplicates, and returns this set of objects. + Splits a string into arrays of sentences, where each sentence is an array of words. + The `language` and `country` arguments are optional, + When they are omitted: + 1.If they are both omitted, the `Locale.ROOT - locale(language='', country='')` is used. + The `Locale.ROOT` is regarded as the base locale of all locales, and is used as the + language/country neutral locale for the locale sensitive operations. + 2.If the `country` is omitted, the `locale(language, country='')` is used. + When they are null: + 1.If they are both `null`, the `Locale.US - locale(language='en', country='US')` is used. + 2.If the `language` is null and the `country` is not null, + the `Locale.US - locale(language='en', country='US')` is used. + 3.If the `language` is not null and the `country` is null, the `locale(language)` is used. + 4.If neither is `null`, the `locale(language, country)` is used. - .. versionadded:: 1.6.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.0.0 + Supports `sentences(string, language)`. + Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the function is computed. + string : :class:`~pyspark.sql.Column` or column name + a string to be split. + A column that evaluates to a string. + language : :class:`~pyspark.sql.Column` or column name, optional + a language of the locale. + A column that evaluates to a string. + country : :class:`~pyspark.sql.Column` or column name, optional + a country of the locale. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing a set of collected values, duplicates excluded. + arrays of split sentences. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.array_agg` - :meth:`pyspark.sql.functions.collect_list` - - Notes - ----- - This function is non-deterministic as the order of collected results depends - on the order of the rows, which may be non-deterministic after any shuffle operations. + :meth:`pyspark.sql.functions.split` + :meth:`pyspark.sql.functions.split_part` Examples -------- - Example 1: Collect values from a DataFrame and sort the result in ascending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) - >>> df.select(sf.sort_array(sf.collect_set('value')).alias('sorted_set')).show() - +----------+ - |sorted_set| - +----------+ - | [1, 2]| - +----------+ - - Example 2: Collect values from a DataFrame and sort the result in descending order - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) - >>> df.select(sf.sort_array(sf.collect_set('age'), asc=False).alias('sorted_set')).show() - +----------+ - |sorted_set| - +----------+ - | [5, 2]| - +----------+ + >>> df = spark.createDataFrame([("This is an example sentence.", )], ["s"]) + >>> df.select("*", sf.sentences(df.s, sf.lit("en"), sf.lit("US"))).show(truncate=False) + +----------------------------+-----------------------------------+ + |s |sentences(s, en, US) | + +----------------------------+-----------------------------------+ + |This is an example sentence.|[[This, is, an, example, sentence]]| + +----------------------------+-----------------------------------+ - Example 3: Collect values from a DataFrame with multiple columns and sort the result + >>> df.select("*", sf.sentences(df.s, sf.lit("en"))).show(truncate=False) + +----------------------------+-----------------------------------+ + |s |sentences(s, en, ) | + +----------------------------+-----------------------------------+ + |This is an example sentence.|[[This, is, an, example, sentence]]| + +----------------------------+-----------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) - >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_set('id')).alias('sorted_set')) - >>> df.orderBy(sf.desc("name")).show() - +----+----------+ - |name|sorted_set| - +----+----------+ - |John| [1, 2]| - | Ana| [3]| - +----+----------+ + >>> df.select("*", sf.sentences(df.s)).show(truncate=False) + +----------------------------+-----------------------------------+ + |s |sentences(s, , ) | + +----------------------------+-----------------------------------+ + |This is an example sentence.|[[This, is, an, example, sentence]]| + +----------------------------+-----------------------------------+ """ - return _invoke_function_over_columns("collect_set", col) + if language is None: + language = lit("") + if country is None: + country = lit("") + + return _invoke_function_over_columns("sentences", string, language, country) @_try_remote_functions -def collect_union(col: "ColumnOrName") -> Column: +def substring( + str: "ColumnOrName", + pos: Union["ColumnOrName", int], + len: Union["ColumnOrName", int], +) -> Column: """ - Aggregate function: given an array-typed column, collects the distinct union of the - elements of the arrays across rows and returns it as an array. + Substring starts at `pos` and is of length `len` when str is String type or + returns the slice of byte array that starts at `pos` in byte and is of length `len` + when str is Binary type. - The aggregation buffer holds only the distinct elements, so its size is bounded by the - element universe rather than by the number of input rows. Null elements are dropped by - default (``IGNORE NULLS``), matching :func:`collect_set`. With ``RESPECT NULLS`` a single - null element is kept, in which case this is equivalent to - ``array_distinct(flatten(collect_list(col)))``. The ``RESPECT NULLS`` clause is only - available through SQL, e.g. ``expr("collect_union(col) RESPECT NULLS")``. + .. versionadded:: 1.5.0 - .. versionadded:: 4.3.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The position is not zero based, but 1 based index. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target array column on which the function is computed. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string or binary. + pos : :class:`~pyspark.sql.Column` or column name or int + starting position in str. + A column that evaluates to an integer. + + .. versionchanged:: 4.0.0 + `pos` now accepts column and column name. + + len : :class:`~pyspark.sql.Column` or column name or int + length of chars. + A column that evaluates to an integer. + + .. versionchanged:: 4.0.0 + `len` now accepts column and column name. Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing the distinct union of the array elements. + substring of given value. + Returns a column of the same type as the input. See Also -------- - :meth:`pyspark.sql.functions.collect_set` - :meth:`pyspark.sql.functions.collect_list` - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.flatten` - - Notes - ----- - This function is non-deterministic as the order of collected results depends - on the order of the rows, which may be non-deterministic after any shuffle operations. + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.locate` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.Column.substr` Examples -------- - Example 1: Union the elements of array columns across rows + Example 1: Using literal integers as arguments - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [([1, 2],), ([2, 3],), ([1],)], ('value',)) - >>> df.select(sf.sort_array(sf.collect_union('value')).alias('u')).show() - +---------+ - | u| - +---------+ - |[1, 2, 3]| - +---------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('abcd',)], ['s',]) + >>> df.select('*', sf.substring(df.s, 1, 2)).show() + +----+------------------+ + | s|substring(s, 1, 2)| + +----+------------------+ + |abcd| ab| + +----+------------------+ - Example 2: Union per group + Example 2: Using columns as arguments - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("a", [1, 2]), ("a", [2, 3]), ("b", [4])], ("k", "value")) - >>> df = df.groupBy("k").agg(sf.sort_array(sf.collect_union('value')).alias('u')) - >>> df.orderBy("k").show() - +---+---------+ - | k| u| - +---+---------+ - | a|[1, 2, 3]| - | b| [4]| - +---+---------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) + >>> df.select('*', sf.substring(df.s, 2, df.l)).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, 2, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + >>> df.select('*', sf.substring(df.s, df.p, 3)).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, p, 3)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + >>> df.select('*', sf.substring(df.s, df.p, df.l)).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, p, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + Example 3: Using column names as arguments + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) + >>> df.select('*', sf.substring(df.s, 2, 'l')).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, 2, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + >>> df.select('*', sf.substring('s', 'p', 'l')).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, p, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ """ - return _invoke_function_over_columns("collect_union", col) + pos = _enum_to_value(pos) + pos = lit(pos) if isinstance(pos, int) else pos + len = _enum_to_value(len) + len = lit(len) if isinstance(len, int) else len + return _invoke_function_over_columns("substring", str, pos, len) @_try_remote_functions -def degrees(col: "ColumnOrName") -> Column: +def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: """ - Converts an angle measured in radians to an approximately equivalent angle - measured in degrees. + Returns the substring from string str before count occurrences of the delimiter delim. + If count is positive, everything the left of the final delimiter (counting from left) is + returned. If count is negative, every to the right of the final delimiter (counting from the + right) is returned. substring_index performs a case-sensitive match when searching for delim. - .. versionadded:: 2.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - angle in radians. - A column that evaluates to a double. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + delim : literal string + delimiter of values. + A column that evaluates to a string. + count : int + number of occurrences. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - angle in degrees, as if computed by `java.lang.Math.toDegrees()` - Returns a column that evaluates to a double. + substring of given value. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.radians` + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.locate` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.Column.substr` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (0.0), (PI()), (PI() / 2), (PI() / 4) AS TAB(value)" - ... ).select("*", sf.degrees("value")).show() - +------------------+--------------+ - | value|DEGREES(value)| - +------------------+--------------+ - | 0.0| 0.0| - | 3.141592653589...| 180.0| - |1.5707963267948...| 90.0| - |0.7853981633974...| 45.0| - +------------------+--------------+ + >>> df = spark.createDataFrame([('a.b.c.d',)], ['s']) + >>> df.select('*', sf.substring_index(df.s, '.', 2)).show() + +-------+------------------------+ + | s|substring_index(s, ., 2)| + +-------+------------------------+ + |a.b.c.d| a.b| + +-------+------------------------+ + + >>> df.select('*', sf.substring_index('s', '.', -3)).show() + +-------+-------------------------+ + | s|substring_index(s, ., -3)| + +-------+-------------------------+ + |a.b.c.d| b.c.d| + +-------+-------------------------+ """ - return _invoke_function_over_columns("degrees", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "substring_index", _to_java_column(str), _enum_to_value(delim), _enum_to_value(count) + ) @_try_remote_functions -def radians(col: "ColumnOrName") -> Column: - """ - Converts an angle measured in degrees to an approximately equivalent angle - measured in radians. +def levenshtein( + left: "ColumnOrName", right: "ColumnOrName", threshold: Optional[int] = None +) -> Column: + """Computes the Levenshtein distance of the two given strings. - .. versionadded:: 2.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - angle in degrees. - A column that evaluates to a double. + left : :class:`~pyspark.sql.Column` or column name + first column value. + A column that evaluates to a string. + right : :class:`~pyspark.sql.Column` or column name + second column value. + A column that evaluates to a string. + threshold : int, optional + if set when the levenshtein distance of the two given strings + less than or equal to a given threshold then return result distance, or -1. + A column that evaluates to an integer. + + .. versionadded:: 3.5.0 Returns ------- :class:`~pyspark.sql.Column` - angle in radians, as if computed by `java.lang.Math.toRadians()` - Returns a column that evaluates to a double. - - See Also - -------- - :meth:`pyspark.sql.functions.degrees` + Levenshtein distance as integer value. + Returns a column that evaluates to an integer. Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (180), (90), (45), (0) AS TAB(value)" - ... ).select("*", sf.radians("value")).show() - +-----+------------------+ - |value| RADIANS(value)| - +-----+------------------+ - | 180| 3.141592653589...| - | 90|1.5707963267948...| - | 45|0.7853981633974...| - | 0| 0.0| - +-----+------------------+ - """ - return _invoke_function_over_columns("radians", col) - + >>> df = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r']) + >>> df.select('*', sf.levenshtein('l', 'r')).show() + +------+-------+-----------------+ + | l| r|levenshtein(l, r)| + +------+-------+-----------------+ + |kitten|sitting| 3| + +------+-------+-----------------+ -@_try_remote_functions -def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: + >>> df.select('*', sf.levenshtein(df.l, df.r, 2)).show() + +------+-------+--------------------+ + | l| r|levenshtein(l, r, 2)| + +------+-------+--------------------+ + |kitten|sitting| -1| + +------+-------+--------------------+ """ - Compute the angle in radians between the positive x-axis of a plane - and the point given by the coordinates + from pyspark.sql.classic.column import _to_java_column - .. versionadded:: 1.4.0 + if threshold is None: + return _invoke_function_over_columns("levenshtein", left, right) + else: + return _invoke_function( + "levenshtein", _to_java_column(left), _to_java_column(right), _enum_to_value(threshold) + ) - .. versionchanged:: 3.4.0 - Supports Spark Connect. + +@_try_remote_functions +def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Computes the Jaro-Winkler similarity between the two given strings. + + The result is a double between 0.0 (no similarity) and 1.0 (identical strings). + + .. versionadded:: 4.3.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column`, column name or float - coordinate on y-axis. - A column that evaluates to a double. - col2 : :class:`~pyspark.sql.Column`, column name or float - coordinate on x-axis. - A column that evaluates to a double. + left : :class:`~pyspark.sql.Column` or column name + first column value. + A column that evaluates to a string. + right : :class:`~pyspark.sql.Column` or column name + second column value. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the `theta` component of the point - (`r`, `theta`) - in polar coordinates that corresponds to the point - (`x`, `y`) in Cartesian coordinates, - as if computed by `java.lang.Math.atan2()` + Jaro-Winkler similarity as a double value. Returns a column that evaluates to a double. - See Also - -------- - :meth:`pyspark.sql.functions.atan` - :meth:`pyspark.sql.functions.hypot` - Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.atan2(sf.lit(1), sf.lit(2))).show() - +------------------+ - | ATAN2(1, 2)| - +------------------+ - |0.4636476090008...| - +------------------+ + >>> df = spark.createDataFrame([('MARTHA', 'MARHTA')], ['l', 'r']) + >>> df.select(sf.jaro_winkler_similarity('l', 'r')).show() + +-----------------------------+ + |jaro_winkler_similarity(l, r)| + +-----------------------------+ + | 0.9611111111111111| + +-----------------------------+ """ - return _invoke_binary_math_function("atan2", col1, col2) + return _invoke_function_over_columns("jaro_winkler_similarity", left, right) @_try_remote_functions -def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: +def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: """ - Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow. + Locate the position of the first occurrence of substr in a string column, after position pos. - .. versionadded:: 1.4.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col1 : :class:`~pyspark.sql.Column`, column name or float - a leg. - A column that evaluates to a double. - col2 : :class:`~pyspark.sql.Column`, column name or float - b leg. - A column that evaluates to a double. + substr : literal string + a string. + A column that evaluates to a string. + str : :class:`~pyspark.sql.Column` or column name + a Column of :class:`pyspark.sql.types.StringType`. + A column that evaluates to a string. + pos : int, optional + start position (zero based). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - length of the hypotenuse. - Returns a column that evaluates to a double. + position of the substring. + Returns a column that evaluates to an integer. + + Notes + ----- + The position is not zero based, but 1 based index. Returns 0 if substr + could not be found in str. + + See Also + -------- + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.Column.substr` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.hypot(sf.lit(1), sf.lit(2))).show() - +----------------+ - | HYPOT(1, 2)| - +----------------+ - |2.23606797749...| - +----------------+ + >>> df = spark.createDataFrame([('abcd',)], ['s',]) + >>> df.select('*', sf.locate('b', 's', 1)).show() + +----+---------------+ + | s|locate(b, s, 1)| + +----+---------------+ + |abcd| 2| + +----+---------------+ + + >>> df.select('*', sf.locate('b', df.s, 3)).show() + +----+---------------+ + | s|locate(b, s, 3)| + +----+---------------+ + |abcd| 0| + +----+---------------+ """ - return _invoke_binary_math_function("hypot", col1, col2) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "locate", _enum_to_value(substr), _to_java_column(str), _enum_to_value(pos) + ) @_try_remote_functions -def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: +def lpad( + col: "ColumnOrName", + len: Union[Column, int], + pad: Union[Column, str], +) -> Column: """ - Returns the value of the first argument raised to the power of the second argument. + Left-pad the string column to width `len` with `pad`. - .. versionadded:: 1.4.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col1 : :class:`~pyspark.sql.Column`, column name or float - the base number. - A column that evaluates to a double. - col2 : :class:`~pyspark.sql.Column`, column name or float - the exponent number. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string or binary. + len : :class:`~pyspark.sql.Column` or int + length of the final string. + A column that evaluates to an integer. + + .. versionchanged:: 4.0.0 + `pattern` now accepts column. + + pad : :class:`~pyspark.sql.Column` or literal string + chars to prepend. + A column that evaluates to a string or binary. + + .. versionchanged:: 4.0.0 + `pattern` now accepts column. Returns ------- :class:`~pyspark.sql.Column` - the base rased to the power the argument. - Returns a column that evaluates to a double. + left padded result. + Returns a column of the same type as the input. + + See Also + -------- + :meth:`pyspark.sql.functions.rpad` Examples -------- + Example 1: Pad with a literal string + >>> from pyspark.sql import functions as sf - >>> spark.range(5).select("*", sf.pow("id", 2)).show() - +---+------------+ - | id|POWER(id, 2)| - +---+------------+ - | 0| 0.0| - | 1| 1.0| - | 2| 4.0| - | 3| 9.0| - | 4| 16.0| - +---+------------+ - """ - return _invoke_binary_math_function("pow", col1, col2) + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.lpad(df.s, 6, '#')).show() + +----+-------------+ + | s|lpad(s, 6, #)| + +----+-------------+ + |abcd| ##abcd| + | xyz| ###xyz| + | 12| ####12| + +----+-------------+ + Example 2: Pad with a bytes column -power = pow + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.lpad(df.s, 6, sf.lit(b"\x75\x76"))).show() + +----+-------------------+ + | s|lpad(s, 6, X'7576')| + +----+-------------------+ + |abcd| uvabcd| + | xyz| uvuxyz| + | 12| uvuv12| + +----+-------------------+ + """ + return _invoke_function_over_columns("lpad", col, lit(len), lit(pad)) @_try_remote_functions -def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", float]) -> Column: +def rpad( + col: "ColumnOrName", + len: Union[Column, int], + pad: Union[Column, str], +) -> Column: """ - Returns the positive value of dividend mod divisor. + Right-pad the string column to width `len` with `pad`. - .. versionadded:: 3.4.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - dividend : :class:`~pyspark.sql.Column`, column name or float - the column that contains dividend, or the specified dividend value. - A column that evaluates to a numeric. - divisor : :class:`~pyspark.sql.Column`, column name or float - the column that contains divisor, or the specified divisor value. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + target column to work on. + A column that evaluates to a string or binary. + len : :class:`~pyspark.sql.Column` or int + length of the final string. + A column that evaluates to an integer. + + .. versionchanged:: 4.0.0 + `pattern` now accepts column. + + pad : :class:`~pyspark.sql.Column` or literal string + chars to prepend. + A column that evaluates to a string or binary. + + .. versionchanged:: 4.0.0 + `pattern` now accepts column. Returns ------- :class:`~pyspark.sql.Column` - positive value of dividend mod divisor. + right padded result. Returns a column of the same type as the input. - Notes - ----- - Supports Spark Connect. + See Also + -------- + :meth:`pyspark.sql.functions.lpad` Examples -------- + Example 1: Pad with a literal string + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1.0, float('nan')), (float('nan'), 2.0), (10.0, 3.0), - ... (float('nan'), float('nan')), (-3.0, 4.0), (-10.0, 3.0), - ... (-5.0, -6.0), (7.0, -8.0), (1.0, 2.0)], - ... ("a", "b")) - >>> df.select("*", sf.pmod("a", "b")).show() - +-----+----+----------+ - | a| b|pmod(a, b)| - +-----+----+----------+ - | 1.0| NaN| NaN| - | NaN| 2.0| NaN| - | 10.0| 3.0| 1.0| - | NaN| NaN| NaN| - | -3.0| 4.0| 1.0| - |-10.0| 3.0| 2.0| - | -5.0|-6.0| -5.0| - | 7.0|-8.0| 7.0| - | 1.0| 2.0| 1.0| - +-----+----+----------+ + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.rpad(df.s, 6, '#')).show() + +----+-------------+ + | s|rpad(s, 6, #)| + +----+-------------+ + |abcd| abcd##| + | xyz| xyz###| + | 12| 12####| + +----+-------------+ + + Example 2: Pad with a bytes column + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.rpad(df.s, 6, sf.lit(b"\x75\x76"))).show() + +----+-------------------+ + | s|rpad(s, 6, X'7576')| + +----+-------------------+ + |abcd| abcduv| + | xyz| xyzuvu| + | 12| 12uvuv| + +----+-------------------+ """ - return _invoke_binary_math_function("pmod", dividend, divisor) + return _invoke_function_over_columns("rpad", col, lit(len), lit(pad)) @_try_remote_functions -def width_bucket( - v: "ColumnOrName", - min: "ColumnOrName", - max: "ColumnOrName", - numBucket: Union["ColumnOrName", int], -) -> Column: +def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: """ - Returns the bucket number into which the value of this expression would fall - after being evaluated. Note that input arguments must follow conditions listed below; - otherwise, the method will return null. + Repeats a string column n times, and returns it as a new string column. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - v : :class:`~pyspark.sql.Column` or column name - value to compute a bucket number in the histogram. - A column that evaluates to a double or interval. - min : :class:`~pyspark.sql.Column` or column name - minimum value of the histogram. - A column that evaluates to a double or interval. - max : :class:`~pyspark.sql.Column` or column name - maximum value of the histogram. - A column that evaluates to a double or interval. - numBucket : :class:`~pyspark.sql.Column`, column name or int - the number of buckets. - A column that evaluates to a long. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + n : :class:`~pyspark.sql.Column` or column name or int + number of times to repeat value. + A column that evaluates to an integer. + + .. versionchanged:: 4.0.0 + `n` now accepts column and column name. Returns ------- :class:`~pyspark.sql.Column` - the bucket number into which the value would fall after being evaluated - Returns a column that evaluates to a long. + string with repeated values. + Returns a column that evaluates to a string. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (5.3, 0.2, 10.6, 5), - ... (-2.1, 1.3, 3.4, 3), - ... (8.1, 0.0, 5.7, 4), - ... (-0.9, 5.2, 0.5, 2)], - ... ['v', 'min', 'max', 'n']) - >>> df.select("*", sf.width_bucket('v', 'min', 'max', 'n')).show() - +----+---+----+---+----------------------------+ - | v|min| max| n|width_bucket(v, min, max, n)| - +----+---+----+---+----------------------------+ - | 5.3|0.2|10.6| 5| 3| - |-2.1|1.3| 3.4| 3| 0| - | 8.1|0.0| 5.7| 4| 5| - |-0.9|5.2| 0.5| 2| 3| - +----+---+----+---+----------------------------+ + Example 1: Repeat with a constant number of times + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ab',)], ['s',]) + >>> df.select("*", sf.repeat("s", 3)).show() + +---+------------+ + | s|repeat(s, 3)| + +---+------------+ + | ab| ababab| + +---+------------+ + + >>> df.select("*", sf.repeat(df.s, sf.lit(4))).show() + +---+------------+ + | s|repeat(s, 4)| + +---+------------+ + | ab| abababab| + +---+------------+ + + Example 2: Repeat with a column containing different number of times + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ab', 5,), ('abc', 6,)], ['s', 't']) + >>> df.select("*", sf.repeat("s", "t")).show() + +---+---+------------------+ + | s| t| repeat(s, t)| + +---+---+------------------+ + | ab| 5| ababababab| + |abc| 6|abcabcabcabcabcabc| + +---+---+------------------+ """ - numBucket = _enum_to_value(numBucket) - numBucket = lit(numBucket) if isinstance(numBucket, int) else numBucket - return _invoke_function_over_columns("width_bucket", v, min, max, numBucket) + n = _enum_to_value(n) + n = lit(n) if isinstance(n, int) else n + return _invoke_function_over_columns("repeat", col, n) @_try_remote_functions -def row_number() -> Column: +def split( + str: "ColumnOrName", + pattern: Union[Column, str], + limit: Union["ColumnOrName", int] = -1, +) -> Column: """ - Window function: returns a sequential number starting at 1 within a window partition. + Splits str around matches of the given pattern. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Returns - ------- - :class:`~pyspark.sql.Column` - the column for calculating row numbers. - - See Also - -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.range(3) - >>> w = Window.orderBy(df.id.desc()) - >>> df.withColumn("desc_order", sf.row_number().over(w)).show() - +---+----------+ - | id|desc_order| - +---+----------+ - | 2| 1| - | 1| 2| - | 0| 3| - +---+----------+ - """ - return _invoke_function("row_number") - - -@_try_remote_functions -def dense_rank() -> Column: - """ - Window function: returns the rank of rows within a window partition, without any gaps. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or column name + a string expression to split. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or literal string + a string representing a regular expression. The regex string should be + a Java regular expression. + A column that evaluates to a string. - The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking - sequence when there are ties. That is, if you were ranking a competition using dense_rank - and had three people tie for second place, you would say that all three were in second - place and that the next person came in third. Rank would give me sequential numbers, making - the person that came in third place (after the ties) would register as coming in fifth. + .. versionchanged:: 4.0.0 + `pattern` now accepts column. Does not accept column name since string type remain + accepted as a regular expression representation, for backwards compatibility. + In addition to int, `limit` now accepts column and column name. - This is equivalent to the DENSE_RANK function in SQL. + limit : :class:`~pyspark.sql.Column` or column name or int + an integer which controls the number of times `pattern` is applied. + A column that evaluates to an integer. - .. versionadded:: 1.6.0 + * ``limit > 0``: The resulting array's length will not be more than `limit`, and the + resulting array's last entry will contain all input beyond the last + matched pattern. + * ``limit <= 0``: `pattern` will be applied as many times as possible, and the resulting + array can be of any size. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionchanged:: 3.0 + `split` now takes an optional `limit` field. If not provided, default limit value is -1. Returns ------- :class:`~pyspark.sql.Column` - the column for calculating ranks. + array of separated strings. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` + :meth:`pyspark.sql.functions.sentences` + :meth:`pyspark.sql.functions.split_part` Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("drank", sf.dense_rank().over(w)).show() - +-----+-----+ - |value|drank| - +-----+-----+ - | 1| 1| - | 1| 1| - | 2| 2| - | 3| 3| - | 3| 3| - | 4| 4| - +-----+-----+ - """ - return _invoke_function("dense_rank") - + Example 1: Repeat with a constant pattern -@_try_remote_functions -def rank() -> Column: - """ - Window function: returns the rank of rows within a window partition. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',]) + >>> df.select('*', sf.split(df.s, '[ABC]')).show() + +--------------+-------------------+ + | s|split(s, [ABC], -1)| + +--------------+-------------------+ + |oneAtwoBthreeC|[one, two, three, ]| + +--------------+-------------------+ - The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking - sequence when there are ties. That is, if you were ranking a competition using dense_rank - and had three people tie for second place, you would say that all three were in second - place and that the next person came in third. Rank would give me sequential numbers, making - the person that came in third place (after the ties) would register as coming in fifth. + >>> df.select('*', sf.split(df.s, '[ABC]', 2)).show() + +--------------+------------------+ + | s|split(s, [ABC], 2)| + +--------------+------------------+ + |oneAtwoBthreeC| [one, twoBthreeC]| + +--------------+------------------+ - This is equivalent to the RANK function in SQL. + >>> df.select('*', sf.split('s', '[ABC]', -2)).show() + +--------------+-------------------+ + | s|split(s, [ABC], -2)| + +--------------+-------------------+ + |oneAtwoBthreeC|[one, two, three, ]| + +--------------+-------------------+ - .. versionadded:: 1.6.0 + Example 2: Repeat with a column containing different patterns and limits - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ('oneAtwoBthreeC', '[ABC]', 2), + ... ('1A2B3C', '[1-9]+', 1), + ... ('aa2bb3cc4', '[1-9]+', -1)], ['s', 'p', 'l']) + >>> df.select('*', sf.split(df.s, df.p)).show() + +--------------+------+---+-------------------+ + | s| p| l| split(s, p, -1)| + +--------------+------+---+-------------------+ + |oneAtwoBthreeC| [ABC]| 2|[one, two, three, ]| + | 1A2B3C|[1-9]+| 1| [, A, B, C]| + | aa2bb3cc4|[1-9]+| -1| [aa, bb, cc, ]| + +--------------+------+---+-------------------+ + + >>> df.select(sf.split('s', df.p, 'l')).show() + +-----------------+ + | split(s, p, l)| + +-----------------+ + |[one, twoBthreeC]| + | [1A2B3C]| + | [aa, bb, cc, ]| + +-----------------+ + """ + limit = _enum_to_value(limit) + limit = lit(limit) if isinstance(limit, int) else limit + return _invoke_function_over_columns("split", str, lit(pattern), limit) + + +@_try_remote_functions +def randstr(length: Union[Column, int], seed: Optional[Union[Column, int]] = None) -> Column: + """Returns a string of the specified length whose characters are chosen uniformly at random from + the following pool of characters: 0-9, a-z, A-Z. The random seed is optional. The string length + must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). + + .. versionadded:: 4.0.0 + + Parameters + ---------- + length : :class:`~pyspark.sql.Column` or int + Number of characters in the string to generate. + A column that evaluates to an integer. Must be a constant. + seed : :class:`~pyspark.sql.Column` or int + Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - the column for calculating ranks. + The generated random string with the specified length. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.row_number` + :meth:`pyspark.sql.functions.rand` + :meth:`pyspark.sql.functions.randn` Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("drank", sf.rank().over(w)).show() - +-----+-----+ - |value|drank| - +-----+-----+ - | 1| 1| - | 1| 1| - | 2| 3| - | 3| 4| - | 3| 4| - | 4| 6| - +-----+-----+ + >>> import pyspark.sql.functions as sf + >>> spark.range(0, 10, 1, 1).select(sf.randstr(16, 3)).show() + +----------------+ + | randstr(16, 3)| + +----------------+ + |nurJIpH4cmmMnsCG| + |fl9YtT5m01trZtIt| + |PD19rAgscTHS7qQZ| + |2CuAICF5UJOruVv4| + |kNZEs8nDpJEoz3Rl| + |OXiU0KN5eaXfjXFs| + |qfnTM1BZAHtN0gBV| + |1p8XiSKwg33KnRPK| + |od5y5MucayQq1bKK| + |tklYPmKmc5sIppWM| + +----------------+ """ - return _invoke_function("rank") + length = _enum_to_value(length) + length = lit(length) + if seed is None: + return _invoke_function_over_columns("randstr", length) + else: + seed = _enum_to_value(seed) + seed = lit(seed) + return _invoke_function_over_columns("randstr", length, seed) @_try_remote_functions -def counter_diff(value: "ColumnOrName", startTime: Optional["ColumnOrName"] = None) -> Column: - """ - Window function: computes the differences between consecutive cumulative counter values in a - time series, thereby converting the counter from the cumulative to the delta format. - - Gracefully handles counter resets by returning NULL. Counter resets are detected when the - counter value decreases, or when the start time advances between rows. - - Use the PARTITION BY clause of the window to separate independent counters. This is done by - specifying all columns which uniquely identify a time series. These are typically the counter - name and any attributes tied to the counter. - - Use the ORDER BY clause of the window to order the observations by the associated timestamp - in ascending order. +def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns a count of the number of times that the Java regex pattern `regexp` is matched + in the string `str`. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - value : :class:`~pyspark.sql.Column` or column name - A cumulative counter. Must be a numeric data type. Must be non-negative. - startTime : :class:`~pyspark.sql.Column` or column name, optional - An optional timestamp parameter which indicates when the counter was last set to zero. - Used to signal counter resets. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - The difference between the current and previous counter value within the window partition. + the number of times that a Java regex pattern is matched in the string. + Returns a column that evaluates to an integer. Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> from datetime import datetime - >>> df = spark.createDataFrame( - ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100), - ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200), - ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400), - ... ('http_requests', datetime(2026, 1, 1, 0, 3), 50), - ... ('http_requests', datetime(2026, 1, 1, 0, 4), 100)], - ... "m STRING, t TIMESTAMP_NTZ, c INT") - >>> w = Window.partitionBy("m").orderBy("t") - >>> df.select("m", "t", "c", sf.counter_diff("c").over(w).alias("diff")).show() - +-------------+-------------------+---+----+ - | m| t| c|diff| - +-------------+-------------------+---+----+ - |http_requests|2026-01-01 00:00:00|100|NULL| - |http_requests|2026-01-01 00:01:00|200| 100| - |http_requests|2026-01-01 00:02:00|400| 200| - |http_requests|2026-01-01 00:03:00| 50|NULL| - |http_requests|2026-01-01 00:04:00|100| 50| - +-------------+-------------------+---+----+ + >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) + >>> df.select('*', sf.regexp_count('str', sf.lit(r'\d+'))).show() + +---------+------+----------------------+ + | str|regexp|regexp_count(str, \d+)| + +---------+------+----------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+----------------------+ - >>> df2 = spark.createDataFrame( - ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100, datetime(2026, 1, 1, 0, 0)), - ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200, datetime(2026, 1, 1, 0, 0)), - ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400, datetime(2026, 1, 1, 0, 0)), - ... ('http_requests', datetime(2026, 1, 1, 0, 3), 500, datetime(2026, 1, 1, 0, 2, 15)), - ... ('http_requests', datetime(2026, 1, 1, 0, 4), 600, datetime(2026, 1, 1, 0, 2, 15))], - ... "m STRING, t TIMESTAMP_NTZ, c INT, s TIMESTAMP_NTZ") - >>> df2.select("m", "t", "s", "c", sf.counter_diff("c", "s").over(w).alias("diff")).show() - +-------------+-------------------+-------------------+---+----+ - | m| t| s| c|diff| - +-------------+-------------------+-------------------+---+----+ - |http_requests|2026-01-01 00:00:00|2026-01-01 00:00:00|100|NULL| - |http_requests|2026-01-01 00:01:00|2026-01-01 00:00:00|200| 100| - |http_requests|2026-01-01 00:02:00|2026-01-01 00:00:00|400| 200| - |http_requests|2026-01-01 00:03:00|2026-01-01 00:02:15|500|NULL| - |http_requests|2026-01-01 00:04:00|2026-01-01 00:02:15|600| 100| - +-------------+-------------------+-------------------+---+----+ + >>> df.select('*', sf.regexp_count('str', sf.lit(r'mmm'))).show() + +---------+------+----------------------+ + | str|regexp|regexp_count(str, mmm)| + +---------+------+----------------------+ + |1a 2b 14m| \d+| 0| + +---------+------+----------------------+ + + >>> df.select('*', sf.regexp_count("str", sf.col("regexp"))).show() + +---------+------+-------------------------+ + | str|regexp|regexp_count(str, regexp)| + +---------+------+-------------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+-------------------------+ + + >>> df.select('*', sf.regexp_count(sf.col('str'), "regexp")).show() + +---------+------+-------------------------+ + | str|regexp|regexp_count(str, regexp)| + +---------+------+-------------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+-------------------------+ """ - if startTime is None: - return _invoke_function_over_columns("counter_diff", value) - return _invoke_function_over_columns("counter_diff", value, startTime) + return _invoke_function_over_columns("regexp_count", str, regexp) @_try_remote_functions -def cume_dist() -> Column: - """ - Window function: returns the cumulative distribution of values within a window partition, - i.e. the fraction of rows that are below the current row. +def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: + r"""Extract a specific group matched by the Java regex `regexp`, from the specified string column. + If the regex did not match, or the specified group did not match, an empty string is returned. - .. versionadded:: 1.6.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + pattern : str + regex pattern to apply. + A column that evaluates to a string. + idx : int + matched group id. + A column that evaluates to an integer. + Returns ------- :class:`~pyspark.sql.Column` - the column for calculating cumulative distribution. + matched value specified by `idx` group id. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` + :meth:`pyspark.sql.functions.regexp_extract_all` Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("cd", sf.cume_dist().over(w)).show() - +-----+---+ - |value| cd| - +-----+---+ - | 1|0.2| - | 2|0.4| - | 3|0.8| - | 3|0.8| - | 4|1.0| - +-----+---+ + >>> df = spark.createDataFrame([('100-200',)], ['str']) + >>> df.select('*', sf.regexp_extract('str', r'(\d+)-(\d+)', 1)).show() + +-------+-----------------------------------+ + | str|regexp_extract(str, (\d+)-(\d+), 1)| + +-------+-----------------------------------+ + |100-200| 100| + +-------+-----------------------------------+ + + >>> df = spark.createDataFrame([('foo',)], ['str']) + >>> df.select('*', sf.regexp_extract('str', r'(\d+)', 1)).show() + +---+-----------------------------+ + |str|regexp_extract(str, (\d+), 1)| + +---+-----------------------------+ + |foo| | + +---+-----------------------------+ + + >>> df = spark.createDataFrame([('aaaac',)], ['str']) + >>> df.select('*', sf.regexp_extract(sf.col('str'), '(a+)(b)?(c)', 2)).show() + +-----+-----------------------------------+ + | str|regexp_extract(str, (a+)(b)?(c), 2)| + +-----+-----------------------------------+ + |aaaac| | + +-----+-----------------------------------+ """ - return _invoke_function("cume_dist") + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "regexp_extract", _to_java_column(str), _enum_to_value(pattern), _enum_to_value(idx) + ) @_try_remote_functions -def percent_rank() -> Column: - """ - Window function: returns the relative rank (i.e. percentile) of rows within a window partition. +def regexp_extract_all( + str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None +) -> Column: + r"""Extract all strings in the `str` that match the Java regex `regexp` + and corresponding to the regex group index. - .. versionadded:: 1.6.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. + idx : :class:`~pyspark.sql.Column` or int, optional + matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - the column for calculating relative rank. + all strings in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` + :meth:`pyspark.sql.functions.regexp_extract` Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("pr", sf.percent_rank().over(w)).show() - +-----+---+ - |value| pr| - +-----+---+ - | 1|0.0| - | 1|0.0| - | 2|0.4| - | 3|0.6| - | 3|0.6| - | 4|1.0| - +-----+---+ - """ - return _invoke_function("percent_rank") - + >>> df = spark.createDataFrame([("100-200, 300-400", r"(\d+)-(\d+)")], ["str", "regexp"]) + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'))).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+---------------------------------------+ -@_try_remote_functions -def approxCountDistinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: - """ - This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates - the approximate distinct count of elements in a specified column or a group of columns. + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), sf.lit(1))).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+---------------------------------------+ - .. versionadded:: 1.3.0 + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), 2)).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 2)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [200, 400]| + +----------------+-----------+---------------------------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> df.select('*', sf.regexp_extract_all('str', sf.col("regexp"))).show() + +----------------+-----------+----------------------------------+ + | str| regexp|regexp_extract_all(str, regexp, 1)| + +----------------+-----------+----------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+----------------------------------+ - .. deprecated:: 2.1.0 - Use :func:`approx_count_distinct` instead. + >>> df.select('*', sf.regexp_extract_all(sf.col('str'), "regexp")).show() + +----------------+-----------+----------------------------------+ + | str| regexp|regexp_extract_all(str, regexp, 1)| + +----------------+-----------+----------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+----------------------------------+ """ - warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", FutureWarning) - return approx_count_distinct(col, rsd) + if idx is None: + return _invoke_function_over_columns("regexp_extract_all", str, regexp) + else: + return _invoke_function_over_columns("regexp_extract_all", str, regexp, lit(idx)) @_try_remote_functions -def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: - """ - This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates - the approximate distinct count of elements in a specified column or a group of columns. +def regexp_replace( + string: "ColumnOrName", + pattern: Union[str, Column], + replacement: Union[str, Column], + position: Optional[Union[int, Column]] = None, +) -> Column: + r"""Replace all substrings of the specified string value that match regexp with replacement. - .. versionadded:: 2.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.3.0 + Supports the `position` parameter. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The label of the column to count distinct values in. - rsd : float, optional - The maximum allowed relative standard deviation (default = 0.05). - If rsd < 0.01, it would be more efficient to use :func:`count_distinct`. + string : :class:`~pyspark.sql.Column` or str + column name or column containing the string value. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or str + column object or str containing the regexp pattern. + A column that evaluates to a string. + replacement : :class:`~pyspark.sql.Column` or str + column object or str containing the replacement. + A column that evaluates to a string. + position : :class:`~pyspark.sql.Column` or int, optional + position to start replacement. The first position is 1. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing the approximate unique count. - - See Also - -------- - :meth:`pyspark.sql.functions.count_distinct` + string with all substrings replaced. + Returns a column that evaluates to a string. Examples -------- - Example 1: Counting distinct values in a single column DataFrame representing integers - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "int") - >>> df.agg(sf.approx_count_distinct("value")).show() - +----------------------------+ - |approx_count_distinct(value)| - +----------------------------+ - | 3| - +----------------------------+ + >>> df = spark.createDataFrame( + ... [("100-200", r"(\d+)", "--")], + ... ["str", "pattern", "replacement"] + ... ) - Example 2: Counting distinct values in a single column DataFrame representing strings + Example 1: Replaces all the substrings in the `str` column name that + match the regex pattern `(\d+)` (one or more digits) with the replacement + string "--". - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("apple",), ("orange",), ("apple",), ("banana",)], ['fruit']) - >>> df.agg(sf.approx_count_distinct("fruit")).show() - +----------------------------+ - |approx_count_distinct(fruit)| - +----------------------------+ - | 3| - +----------------------------+ + >>> df.select('*', sf.regexp_replace('str', r'(\d+)', '--')).show() + +-------+-------+-----------+---------------------------------+ + | str|pattern|replacement|regexp_replace(str, (\d+), --, 1)| + +-------+-------+-----------+---------------------------------+ + |100-200| (\d+)| --| -----| + +-------+-------+-----------+---------------------------------+ - Example 3: Counting distinct values in a DataFrame with multiple columns + Example 2: Replaces all the substrings in the `str` Column that match + the regex pattern in the `pattern` Column with the string in the `replacement` + column. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("Alice", 1), ("Alice", 2), ("Bob", 3), ("Bob", 3)], ["name", "value"]) - >>> df = df.withColumn("combined", sf.struct("name", "value")) - >>> df.agg(sf.approx_count_distinct(df.combined)).show() - +-------------------------------+ - |approx_count_distinct(combined)| - +-------------------------------+ - | 3| - +-------------------------------+ + >>> df.select('*', \ + ... sf.regexp_replace(sf.col("str"), sf.col("pattern"), sf.col("replacement")) \ + ... ).show() + +-------+-------+-----------+--------------------------------------------+ + | str|pattern|replacement|regexp_replace(str, pattern, replacement, 1)| + +-------+-------+-----------+--------------------------------------------+ + |100-200| (\d+)| --| -----| + +-------+-------+-----------+--------------------------------------------+ - Example 4: Counting distinct values with a specified relative standard deviation + Example 3: Replaces substrings starting from the specified position. + For the input string "100-200", position 5 starts replacement after "100-". - >>> from pyspark.sql import functions as sf - >>> spark.range(100000).agg( - ... sf.approx_count_distinct("id").alias('with_default_rsd'), - ... sf.approx_count_distinct("id", 0.1).alias('with_rsd_0.1') - ... ).show() - +----------------+------------+ - |with_default_rsd|with_rsd_0.1| - +----------------+------------+ - | 95546| 102065| - +----------------+------------+ + >>> df.select(sf.regexp_replace("str", r"(\d+)", "--", 5).alias("d")).show() + +------+ + | d| + +------+ + |100---| + +------+ """ - from pyspark.sql.classic.column import _to_java_column - - if rsd is None: - return _invoke_function_over_columns("approx_count_distinct", col) + if position is None: + return _invoke_function_over_columns( + "regexp_replace", string, lit(pattern), lit(replacement) + ) else: - return _invoke_function("approx_count_distinct", _to_java_column(col), _enum_to_value(rsd)) + return _invoke_function_over_columns( + "regexp_replace", + string, + lit(pattern), + lit(replacement), + lit(position), + ) @_try_remote_functions -def broadcast(df: "DataFrame") -> "DataFrame": - """ - Marks a DataFrame as small enough for use in broadcast joins. +def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns the first substring that matches the Java regex `regexp` within the string `str`. + If the regular expression is not found, the result is null. - .. versionadded:: 1.6.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. Returns ------- - :class:`~pyspark.sql.DataFrame` - DataFrame marked as ready for broadcast join. + :class:`~pyspark.sql.Column` + the first substring that matches a Java regex within the string `str`. + Returns a column that evaluates to a string. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") - >>> df_small = spark.range(3) - >>> df_b = sf.broadcast(df_small) - >>> df.join(df_b, df.value == df_small.id).show() - +-----+---+ - |value| id| - +-----+---+ - | 1| 1| - | 2| 2| - +-----+---+ - """ - from py4j.java_gateway import JVMView - - from pyspark.sql.dataframe import DataFrame - - sc = _get_active_spark_context() - return DataFrame(cast(JVMView, sc._jvm).functions.broadcast(df._jdf), df.sparkSession) - + >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) -@_try_remote_functions -def coalesce(*cols: "ColumnOrName") -> Column: - """Returns the first column that is not null. + Example 1: Returns the first substring in the `str` column name that + matches the regex pattern `(\d+)` (one or more digits). - .. versionadded:: 1.4.0 + >>> df.select('*', sf.regexp_substr('str', sf.lit(r'\d+'))).show() + +---------+------+-----------------------+ + | str|regexp|regexp_substr(str, \d+)| + +---------+------+-----------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+-----------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 2: Returns the first substring in the `str` column name that + matches the regex pattern `(mmm)` (three consecutive 'm' characters) - Parameters - ---------- - cols : :class:`~pyspark.sql.Column` or column name - list of columns to work on. - Each a column of any type. + >>> df.select('*', sf.regexp_substr('str', sf.lit(r'mmm'))).show() + +---------+------+-----------------------+ + | str|regexp|regexp_substr(str, mmm)| + +---------+------+-----------------------+ + |1a 2b 14m| \d+| NULL| + +---------+------+-----------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - value of the first column that is not null. - Returns a column of the same type as the input. + Example 3: Returns the first substring in the `str` column name that + matches the regex pattern in `regexp` Column. - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None, None), (1, None), (None, 2)], ("a", "b")) - >>> df.show() - +----+----+ - | a| b| - +----+----+ - |NULL|NULL| - | 1|NULL| - |NULL| 2| - +----+----+ + >>> df.select('*', sf.regexp_substr("str", sf.col("regexp"))).show() + +---------+------+--------------------------+ + | str|regexp|regexp_substr(str, regexp)| + +---------+------+--------------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+--------------------------+ - >>> df.select('*', sf.coalesce("a", df["b"])).show() - +----+----+--------------+ - | a| b|coalesce(a, b)| - +----+----+--------------+ - |NULL|NULL| NULL| - | 1|NULL| 1| - |NULL| 2| 2| - +----+----+--------------+ + Example 4: Returns the first substring in the `str` Column that + matches the regex pattern in `regexp` column name. - >>> df.select('*', sf.coalesce(df["a"], lit(0.0))).show() - +----+----+----------------+ - | a| b|coalesce(a, 0.0)| - +----+----+----------------+ - |NULL|NULL| 0.0| - | 1|NULL| 1.0| - |NULL| 2| 0.0| - +----+----+----------------+ + >>> df.select('*', sf.regexp_substr(sf.col("str"), "regexp")).show() + +---------+------+--------------------------+ + | str|regexp|regexp_substr(str, regexp)| + +---------+------+--------------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+--------------------------+ """ - return _invoke_function_over_seq_of_columns("coalesce", cols) + return _invoke_function_over_columns("regexp_substr", str, regexp) @_try_remote_functions -def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for - ``col1`` and ``col2``. - - .. versionadded:: 1.6.0 +def regexp_instr( + str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None +) -> Column: + r"""Returns the position of the first substring in the `str` that match the Java regex `regexp` + and corresponding to the regex group index. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to calculate correlation. - A column that evaluates to a numeric. - col2 : :class:`~pyspark.sql.Column` or column name - second column to calculate correlation. - A column that evaluates to a numeric. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. + idx : :class:`~pyspark.sql.Column` or int, optional + matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - Pearson Correlation Coefficient of these two column values. + the position of the first substring in the `str` that match a Java regex and corresponding + to the regex group index. + Returns a column that evaluates to an integer. Examples -------- >>> from pyspark.sql import functions as sf - >>> a = range(20) - >>> b = [2 * x for x in range(20)] - >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) - >>> df.agg(sf.corr("a", df.b)).show() - +----------+ - |corr(a, b)| - +----------+ - | 1.0| - +----------+ - """ - return _invoke_function_over_columns("corr", col1, col2) + >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+(a|b|m)")], ["str", "regexp"]) + Example 1: Returns the position of the first substring in the `str` column name that + match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'). -@_try_remote_functions -def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and - ``col2``. + >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'))).show() + +---------+----------+--------------------------------+ + | str| regexp|regexp_instr(str, \d+(a|b|m), 0)| + +---------+----------+--------------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+--------------------------------+ - .. versionadded:: 2.0.0 + Example 2: Returns the position of the first substring in the `str` column name that + match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'), - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'), sf.lit(1))).show() + +---------+----------+--------------------------------+ + | str| regexp|regexp_instr(str, \d+(a|b|m), 1)| + +---------+----------+--------------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+--------------------------------+ - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to calculate covariance. - A column that evaluates to a numeric. - col2 : :class:`~pyspark.sql.Column` or column name - second column to calculate covariance. - A column that evaluates to a numeric. + Example 3: Returns the position of the first substring in the `str` column name that + match the regex pattern in `regexp` Column. - Returns - ------- - :class:`~pyspark.sql.Column` - covariance of these two column values. + >>> df.select('*', sf.regexp_instr('str', sf.col("regexp"))).show() + +---------+----------+----------------------------+ + | str| regexp|regexp_instr(str, regexp, 0)| + +---------+----------+----------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+----------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.covar_samp` + Example 4: Returns the position of the first substring in the `str` Column that + match the regex pattern in `regexp` column name. - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> a = [1] * 10 - >>> b = [1] * 10 - >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) - >>> df.agg(sf.covar_pop("a", df.b)).show() - +---------------+ - |covar_pop(a, b)| - +---------------+ - | 0.0| - +---------------+ + >>> df.select('*', sf.regexp_instr(sf.col("str"), "regexp")).show() + +---------+----------+----------------------------+ + | str| regexp|regexp_instr(str, regexp, 0)| + +---------+----------+----------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+----------------------------+ """ - return _invoke_function_over_columns("covar_pop", col1, col2) + if idx is None: + return _invoke_function_over_columns("regexp_instr", str, regexp) + else: + return _invoke_function_over_columns("regexp_instr", str, regexp, lit(idx)) @_try_remote_functions -def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and - ``col2``. +def initcap(col: "ColumnOrName") -> Column: + """Translate the first letter of each word to upper case in the sentence. - .. versionadded:: 2.0.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to calculate covariance. - A column that evaluates to a numeric. - col2 : :class:`~pyspark.sql.Column` or column name - second column to calculate covariance. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - sample covariance of these two column values. - - See Also - -------- - :meth:`pyspark.sql.functions.covar_pop` + string with all first letters are uppercase in each word. + Returns a column that evaluates to a string. Examples -------- - >>> from pyspark.sql import functions as sf - >>> a = [1] * 10 - >>> b = [1] * 10 - >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) - >>> df.agg(sf.covar_samp("a", df.b)).show() - +----------------+ - |covar_samp(a, b)| - +----------------+ - | 0.0| - +----------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ab cd',)], ['a']) + >>> df.select("*", sf.initcap("a")).show() + +-----+----------+ + | a|initcap(a)| + +-----+----------+ + |ab cd| Ab Cd| + +-----+----------+ """ - return _invoke_function_over_columns("covar_samp", col1, col2) + return _invoke_function_over_columns("initcap", col) @_try_remote_functions -def countDistinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``. - - An alias of :func:`count_distinct`, and it is encouraged to use :func:`count_distinct` - directly. +def soundex(col: "ColumnOrName") -> Column: + """ + Returns the SoundEx encoding for a string - .. versionadded:: 1.3.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + SoundEx encoded string. + Returns a column that evaluates to a string. + Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) - >>> df.select(sf.count_distinct(df.value)).show() - +---------------------+ - |count(DISTINCT value)| - +---------------------+ - | 2| - +---------------------+ - - >>> df.select(sf.countDistinct(df.value)).show() - +---------------------+ - |count(DISTINCT value)| - +---------------------+ - | 2| - +---------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Peters",),("Uhrbach",)], ["s"]) + >>> df.select("*", sf.soundex("s")).show() + +-------+----------+ + | s|soundex(s)| + +-------+----------+ + | Peters| P362| + |Uhrbach| U612| + +-------+----------+ """ - return count_distinct(col, *cols) + return _invoke_function_over_columns("soundex", col) @_try_remote_functions -def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: - """Returns a new :class:`Column` for distinct count of ``col`` or ``cols``. +def length(col: "ColumnOrName") -> Column: + """Computes the character length of string data or number of bytes of binary data. + The length of character data includes the trailing spaces. The length of binary data + includes binary zeros. - .. versionadded:: 3.2.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -7385,128 +7540,76 @@ def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - first column to compute on. - cols : :class:`~pyspark.sql.Column` or column name - other columns to compute on. + target column to work on. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` - distinct values of these two column values. + length of the value. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.approx_count_distinct` + :meth:`pyspark.sql.functions.char_length` + :meth:`pyspark.sql.functions.character_length` Examples -------- - Example 1: Counting distinct values of a single column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) - >>> df.select(sf.count_distinct(df.value)).show() - +---------------------+ - |count(DISTINCT value)| - +---------------------+ - | 2| - +---------------------+ - - Example 2: Counting distinct values of multiple columns - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) - >>> df.select(sf.count_distinct(df.value1, df.value2)).show() - +------------------------------+ - |count(DISTINCT value1, value2)| - +------------------------------+ - | 2| - +------------------------------+ - - Example 3: Counting distinct values with column names as strings - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) - >>> df.select(sf.count_distinct("value1", "value2")).show() - +------------------------------+ - |count(DISTINCT value1, value2)| - +------------------------------+ - | 2| - +------------------------------+ + >>> spark.createDataFrame([('ABC ',)], ['a']).select('*', sf.length('a')).show() + +----+---------+ + | a|length(a)| + +----+---------+ + |ABC | 4| + +----+---------+ """ - from pyspark.sql.classic.column import _to_java_column, _to_seq - - sc = _get_active_spark_context() - return _invoke_function( - "count_distinct", _to_java_column(col), _to_seq(sc, cols, _to_java_column) - ) + return _invoke_function_over_columns("length", col) @_try_remote_functions -def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: - """Aggregate function: returns the first value in a group. - - The function by default returns the first values it sees. It will return the first non-null - value it sees when ignoreNulls is set to true. If all values are null, then null is returned. +def octet_length(col: "ColumnOrName") -> Column: + """ + Calculates the byte length for the specified string column. - .. versionadded:: 1.3.0 + .. versionadded:: 3.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Notes - ----- - The function is non-deterministic because its results depends on the order of the - rows which may be non-deterministic after a shuffle. - Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column to fetch first value for. - A column of any type. - ignorenulls : bool - if first value is null then look for first non-null value. ``False`` by default. - A column that evaluates to a boolean. Must be a constant. + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` - first value of the group. + Byte length of the col + Returns a column that evaluates to an integer. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) - >>> df = df.orderBy(df.age) - >>> df.groupby("name").agg(sf.first("age")).orderBy("name").show() - +-----+----------+ - | name|first(age)| - +-----+----------+ - |Alice| NULL| - | Bob| 5| - +-----+----------+ - - To ignore any null values, set ``ignorenulls`` to `True` - - >>> df.groupby("name").agg(sf.first("age", ignorenulls=True)).orderBy("name").show() - +-----+----------+ - | name|first(age)| - +-----+----------+ - |Alice| 2| - | Bob| 5| - +-----+----------+ + >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) + >>> df.select('*', sf.octet_length('cat')).show() + +---+-----------------+ + |cat|octet_length(cat)| + +---+-----------------+ + |cat| 3| + | 🐈| 4| + +---+-----------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("first", _to_java_column(col), _enum_to_value(ignorenulls)) + return _invoke_function_over_columns("octet_length", col) @_try_remote_functions -def grouping(col: "ColumnOrName") -> Column: +def bit_length(col: "ColumnOrName") -> Column: """ - Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated - or not, returns 1 for aggregated or 0 for not aggregated in the result set. + Calculates the bit length for the specified string column. - .. versionadded:: 2.0.0 + .. versionadded:: 3.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -7514,1721 +7617,1466 @@ def grouping(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column to check if it's aggregated. + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` - returns 1 for aggregated or 0 for not aggregated in the result set. + Bit length of the col + Returns a column that evaluates to an integer. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) - >>> df.cube("name").agg(sf.grouping("name"), sf.sum("age")).orderBy("name").show() - +-----+--------------+--------+ - | name|grouping(name)|sum(age)| - +-----+--------------+--------+ - | NULL| 1| 7| - |Alice| 0| 2| - | Bob| 0| 5| - +-----+--------------+--------+ + >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) + >>> df.select('*', sf.bit_length('cat')).show() + +---+---------------+ + |cat|bit_length(cat)| + +---+---------------+ + |cat| 24| + | 🐈| 32| + +---+---------------+ """ - return _invoke_function_over_columns("grouping", col) + return _invoke_function_over_columns("bit_length", col) @_try_remote_functions -def grouping_id(*cols: "ColumnOrName") -> Column: - """ - Aggregate function: returns the level of grouping, equals to - - (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn) +def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: + """A function translate any character in the `srcCol` by a character in `matching`. + The characters in `replace` is corresponding to the characters in `matching`. + Translation will happen whenever any character in the string is matching with the character + in the `matching`. - .. versionadded:: 2.0.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Notes - ----- - The list of columns should match with grouping columns exactly, or empty (means all - the grouping columns). - Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - columns to check for. + srcCol : :class:`~pyspark.sql.Column` or column name + Source column or strings. + A column that evaluates to a string. + matching : str + matching characters. + A column that evaluates to a string. + replace : str + characters for replacement. If this is shorter than `matching` string then + those chars that don't have replacement will be dropped. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - returns level of the grouping it relates to. + replaced value. + Returns a column that evaluates to a string. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, "a", "a"), (3, "a", "a"), (4, "b", "c")], ["c1", "c2", "c3"]) - >>> df.cube("c2", "c3").agg(sf.grouping_id(), sf.sum("c1")).orderBy("c2", "c3").show() - +----+----+-------------+-------+ - | c2| c3|grouping_id()|sum(c1)| - +----+----+-------------+-------+ - |NULL|NULL| 3| 8| - |NULL| a| 2| 4| - |NULL| c| 2| 4| - | a|NULL| 1| 4| - | a| a| 0| 4| - | b|NULL| 1| 4| - | b| c| 0| 4| - +----+----+-------------+-------+ + >>> df = spark.createDataFrame([('translate',)], ['a']) + >>> df.select('*', sf.translate('a', "rnlt", "123")).show() + +---------+-----------------------+ + | a|translate(a, rnlt, 123)| + +---------+-----------------------+ + |translate| 1a2s3ae| + +---------+-----------------------+ """ - return _invoke_function_over_seq_of_columns("grouping_id", cols) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "translate", _to_java_column(srcCol), _enum_to_value(matching), _enum_to_value(replace) + ) @_try_remote_functions -def count_min_sketch( - col: "ColumnOrName", - eps: Union[Column, float], - confidence: Union[Column, float], - seed: Optional[Union[Column, int]] = None, -) -> Column: +def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: """ - Returns a count-min sketch of a column with the given esp, confidence and seed. - The result is an array of bytes, which can be deserialized to a `CountMinSketch` before usage. - Count-min sketch is a probabilistic data structure used for cardinality estimation - using sub-linear space. + Converts the input `col` to a binary value based on the supplied `format`. + The `format` can be a case-insensitive string literal of "hex", "utf-8", "utf8", + or "base64". By default, the binary format for conversion is "hex" if + `format` is omitted. The function returns NULL if at least one of the + input parameters is NULL. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - eps : :class:`~pyspark.sql.Column` or float - relative error, must be positive - - .. versionchanged:: 4.0.0 - `eps` now accepts float value. - - confidence : :class:`~pyspark.sql.Column` or float - confidence, must be positive and less than 1.0 - - .. versionchanged:: 4.0.0 - `confidence` now accepts float value. - - seed : :class:`~pyspark.sql.Column` or int, optional - random seed - - .. versionchanged:: 4.0.0 - `seed` now accepts int value. + col : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert binary values. + A column that evaluates to a string. Must be a constant. - Returns - ------- - :class:`~pyspark.sql.Column` - count-min sketch of the column + See Also + -------- + :meth:`pyspark.sql.functions.try_to_binary` Examples -------- - Example 1: Using columns as arguments - - >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch(sf.col("id"), sf.lit(3.0), sf.lit(0.1), sf.lit(1))) - ... ).show(truncate=False) - +------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 3.0, 0.1, 1)) | - +------------------------------------------------------------------------+ - |0000000100000000000000640000000100000001000000005D8D6AB90000000000000064| - +------------------------------------------------------------------------+ - - Example 2: Using numbers as arguments - - >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch("id", 1.0, 0.3, 2)) - ... ).show(truncate=False) - +----------------------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 1.0, 0.3, 2)) | - +----------------------------------------------------------------------------------------+ - |0000000100000000000000640000000100000002000000005D96391C00000000000000320000000000000032| - +----------------------------------------------------------------------------------------+ - - Example 3: Using a long seed + Example 1: Convert string to a binary with encoding specified - >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.2, 1111111111111111111)) - ... ).show(truncate=False) - +----------------------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 1.5, 0.2, 1111111111111111111)) | - +----------------------------------------------------------------------------------------+ - |00000001000000000000006400000001000000020000000044078BA100000000000000320000000000000032| - +----------------------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("abc",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() + [Row(r=b'abc')] - Example 4: Using a random seed + Example 2: Convert string to a timestamp without encoding specified - >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.6)) - ... ).show(truncate=False) # doctest: +SKIP - +----------------------------------------------------------------------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 1.5, 0.6, 2120704260)) | - +----------------------------------------------------------------------------------------------------------------------------------------+ - |0000000100000000000000640000000200000002000000005ADECCEE00000000153EBE090000000000000033000000000000003100000000000000320000000000000032| - +----------------------------------------------------------------------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("414243",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() + [Row(r=b'ABC')] """ - _eps = lit(eps) - _conf = lit(confidence) - if seed is None: - return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf) + if format is not None: + return _invoke_function_over_columns("to_binary", col, format) else: - return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf, lit(seed)) + return _invoke_function_over_columns("to_binary", col) @_try_remote_functions -def input_file_name() -> Column: +def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: """ - Creates a string column for the file name of the current Spark task. - - .. versionadded:: 1.6.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Convert `col` to a string based on the `format`. + Throws an exception if the conversion fails. The format can consist of the following + characters, case insensitive: + '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the + format string matches a sequence of digits in the input value, generating a result + string of the same length as the corresponding sequence in the format string. + The result string is left-padded with zeros if the 0/9 sequence comprises more digits + than the matching part of the decimal value, starts with 0, and is before the decimal + point. Otherwise, it is padded with spaces. + '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). + ',' or 'G': Specifies the position of the grouping (thousands) separator (,). + There must be a 0 or 9 to the left and right of each grouping separator. + '$': Specifies the location of the $ currency sign. This character may only be specified once. + 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at + the beginning or end of the format string). Note that 'S' prints '+' for positive + values but 'MI' prints a space. + 'PR': Only allowed at the end of the format string; specifies that the result string + will be wrapped by angle brackets if the input value is negative. + If `col` is a datetime, `format` shall be a valid datetime pattern, see + Patterns. + If `col` is a binary, it is converted to a string in one of the formats: + 'base64': a base 64 string. + 'hex': a string in the hexadecimal format. + 'utf-8': the input binary is decoded to UTF-8 string. - Returns - ------- - :class:`~pyspark.sql.Column` - file names. + .. versionadded:: 3.5.0 - See Also - -------- - :meth:`pyspark.sql.functions.input_file_block_length` - :meth:`pyspark.sql.functions.input_file_block_start` + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert char values. + A column that evaluates to a string. Must be a constant when ``col`` is numeric + or binary. Examples -------- - >>> import os - >>> from pyspark.sql import functions as sf - >>> path = os.path.abspath(__file__) - >>> df = spark.read.text(path) - >>> df.select(sf.input_file_name()).first() - Row(input_file_name()='file:///...') + >>> df = spark.createDataFrame([(78.12,)], ["e"]) + >>> df.select(to_char(df.e, lit("$99.99")).alias('r')).collect() + [Row(r='$78.12')] """ - return _invoke_function("input_file_name") + return _invoke_function_over_columns("to_char", col, format) @_try_remote_functions -def isnan(col: "ColumnOrName") -> Column: - """An expression that returns true if the column is NaN. - - .. versionadded:: 1.6.0 +def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: + """ + Convert `col` to a string based on the `format`. + Throws an exception if the conversion fails. The format can consist of the following + characters, case insensitive: + '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the + format string matches a sequence of digits in the input value, generating a result + string of the same length as the corresponding sequence in the format string. + The result string is left-padded with zeros if the 0/9 sequence comprises more digits + than the matching part of the decimal value, starts with 0, and is before the decimal + point. Otherwise, it is padded with spaces. + '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). + ',' or 'G': Specifies the position of the grouping (thousands) separator (,). + There must be a 0 or 9 to the left and right of each grouping separator. + '$': Specifies the location of the $ currency sign. This character may only be specified once. + 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at + the beginning or end of the format string). Note that 'S' prints '+' for positive + values but 'MI' prints a space. + 'PR': Only allowed at the end of the format string; specifies that the result string + will be wrapped by angle brackets if the input value is negative. + If `col` is a datetime, `format` shall be a valid datetime pattern, see + Patterns. + If `col` is a binary, it is converted to a string in one of the formats: + 'base64': a base 64 string. + 'hex': a string in the hexadecimal format. + 'utf-8': the input binary is decoded to UTF-8 string. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a double or float. - - Returns - ------- - :class:`~pyspark.sql.Column` - True if value is NaN and False otherwise. - Returns a column that evaluates to a boolean. - - See Also - -------- - :meth:`pyspark.sql.functions.isnull` - :meth:`pyspark.sql.functions.isnotnull` + col : :class:`~pyspark.sql.Column` or str + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert char values. + A column that evaluates to a string. Must be a constant when ``col`` is numeric + or binary. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) - >>> df.select("*", sf.isnan("a"), sf.isnan(df.b)).show() - +---+---+--------+--------+ - | a| b|isnan(a)|isnan(b)| - +---+---+--------+--------+ - |1.0|NaN| false| true| - |NaN|2.0| true| false| - +---+---+--------+--------+ + >>> df = spark.createDataFrame([(78.12,)], ["e"]) + >>> df.select(to_varchar(df.e, lit("$99.99")).alias('r')).collect() + [Row(r='$78.12')] """ - return _invoke_function_over_columns("isnan", col) + return _invoke_function_over_columns("to_varchar", col, format) @_try_remote_functions -def isnull(col: "ColumnOrName") -> Column: - """An expression that returns true if the column is null. - - .. versionadded:: 1.6.0 +def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: + """ + Convert string 'col' to a number based on the string format 'format'. + Throws an exception if the conversion fails. The format can consist of the following + characters, case insensitive: + '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the + format string matches a sequence of digits in the input string. If the 0/9 + sequence starts with 0 and is before the decimal point, it can only match a digit + sequence of the same size. Otherwise, if the sequence starts with 9 or is after + the decimal point, it can match a digit sequence that has the same or smaller size. + '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). + ',' or 'G': Specifies the position of the grouping (thousands) separator (,). + There must be a 0 or 9 to the left and right of each grouping separator. + 'col' must match the grouping separator relevant for the size of the number. + '$': Specifies the location of the $ currency sign. This character may only be + specified once. + 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed + once at the beginning or end of the format string). Note that 'S' allows '-' + but 'MI' does not. + 'PR': Only allowed at the end of the format string; specifies that 'col' indicates a + negative number with wrapping angled brackets. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column of any type. - - Returns - ------- - :class:`~pyspark.sql.Column` - True if value is null and False otherwise. - Returns a column that evaluates to a boolean. + col : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert number values. + A column that evaluates to a string. Must be a constant. See Also -------- - :meth:`pyspark.sql.functions.isnan` - :meth:`pyspark.sql.functions.isnotnull` + :meth:`pyspark.sql.functions.try_to_number` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, None), (None, 2)], ("a", "b")) - >>> df.select("*", sf.isnull("a"), isnull(df.b)).show() - +----+----+-----------+-----------+ - | a| b|(a IS NULL)|(b IS NULL)| - +----+----+-----------+-----------+ - | 1|NULL| false| true| - |NULL| 2| true| false| - +----+----+-----------+-----------+ + >>> df = spark.createDataFrame([("$78.12",)], ["e"]) + >>> df.select(to_number(df.e, lit("$99.99")).alias('r')).collect() + [Row(r=Decimal('78.12'))] """ - return _invoke_function_over_columns("isnull", col) + return _invoke_function_over_columns("to_number", col, format) @_try_remote_functions -def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: - """Aggregate function: returns the last value in a group. - - The function by default returns the last values it sees. It will return the last non-null - value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. +def replace( + src: "ColumnOrName", search: "ColumnOrName", replace: Optional["ColumnOrName"] = None +) -> Column: + """ + Replaces all occurrences of `search` with `replace`. - Notes - ----- - The function is non-deterministic because its results depends on the order of the - rows which may be non-deterministic after a shuffle. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to fetch last value for. - A column of any type. - ignorenulls : bool - if last value is null then look for non-null value. ``False`` by default. - A column that evaluates to a boolean. Must be a constant. - - Returns - ------- - :class:`~pyspark.sql.Column` - last value of the group. + src : :class:`~pyspark.sql.Column` or str + A column of string to be replaced. + A column that evaluates to a string. + search : :class:`~pyspark.sql.Column` or str + A column of string, If `search` is not found in `str`, `str` is returned unchanged. + A column that evaluates to a string. + replace : :class:`~pyspark.sql.Column` or str, optional + A column of string, If `replace` is not specified or is an empty string, + nothing replaces the string that is removed from `str`. + A column that evaluates to a string. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) - >>> df = df.orderBy(df.age.desc()) - >>> df.groupby("name").agg(sf.last("age")).orderBy("name").show() - +-----+---------+ - | name|last(age)| - +-----+---------+ - |Alice| NULL| - | Bob| 5| - +-----+---------+ - - To ignore any null values, set ``ignorenulls`` to `True` + >>> df = spark.createDataFrame([("ABCabc", "abc", "DEF",)], ["a", "b", "c"]) + >>> df.select(replace(df.a, df.b, df.c).alias('r')).collect() + [Row(r='ABCDEF')] - >>> df.groupby("name").agg(sf.last("age", ignorenulls=True)).orderBy("name").show() - +-----+---------+ - | name|last(age)| - +-----+---------+ - |Alice| 2| - | Bob| 5| - +-----+---------+ + >>> df.select(replace(df.a, df.b).alias('r')).collect() + [Row(r='ABC')] """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("last", _to_java_column(col), _enum_to_value(ignorenulls)) + if replace is not None: + return _invoke_function_over_columns("replace", src, search, replace) + else: + return _invoke_function_over_columns("replace", src, search) @_try_remote_functions -def monotonically_increasing_id() -> Column: - """A column that generates monotonically increasing 64-bit integers. - - The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. - The current implementation puts the partition ID in the upper 31 bits, and the record number - within each partition in the lower 33 bits. The assumption is that the data frame has - less than 1 billion partitions, and each partition has less than 8 billion records. - - .. versionadded:: 1.6.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. +def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnOrName") -> Column: + """ + Splits `str` by delimiter and return requested part of the split (1-based). + If any input is null, returns null. if `partNum` is out of range of split parts, + returns empty string. If `partNum` is 0, throws an error. If `partNum` is negative, + the parts are counted backward from the end of the string. + If the `delimiter` is an empty string, the `str` is not split. - Notes - ----- - The function is non-deterministic because its result depends on partition IDs. + .. versionadded:: 3.5.0 - As an example, consider a :class:`DataFrame` with two partitions, each with 3 records. - This expression would return the following IDs: - 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. + Parameters + ---------- + src : :class:`~pyspark.sql.Column` or column name + A column of string to be split. + A column that evaluates to a string. + delimiter : :class:`~pyspark.sql.Column` or column name + A column of string, the delimiter used for split. + A column that evaluates to a string. + partNum : :class:`~pyspark.sql.Column` or column name + The requested part of the split (1-based). + A column that evaluates to an integer. - Returns - ------- - :class:`~pyspark.sql.Column` - last value of the group. + See Also + -------- + :meth:`pyspark.sql.functions.sentences` + :meth:`pyspark.sql.functions.split` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(0, 10, 1, 2).select( - ... "*", - ... sf.spark_partition_id(), - ... sf.monotonically_increasing_id()).show() - +---+--------------------+-----------------------------+ - | id|SPARK_PARTITION_ID()|monotonically_increasing_id()| - +---+--------------------+-----------------------------+ - | 0| 0| 0| - | 1| 0| 1| - | 2| 0| 2| - | 3| 0| 3| - | 4| 0| 4| - | 5| 1| 8589934592| - | 6| 1| 8589934593| - | 7| 1| 8589934594| - | 8| 1| 8589934595| - | 9| 1| 8589934596| - +---+--------------------+-----------------------------+ + >>> df = spark.createDataFrame([("11.12.13", ".", 3,)], ["a", "b", "c"]) + >>> df.select("*", sf.split_part("a", "b", "c")).show() + +--------+---+---+-------------------+ + | a| b| c|split_part(a, b, c)| + +--------+---+---+-------------------+ + |11.12.13| .| 3| 13| + +--------+---+---+-------------------+ + + >>> df.select("*", sf.split_part(df.a, df.b, sf.lit(-2))).show() + +--------+---+---+--------------------+ + | a| b| c|split_part(a, b, -2)| + +--------+---+---+--------------------+ + |11.12.13| .| 3| 12| + +--------+---+---+--------------------+ """ - return _invoke_function("monotonically_increasing_id") + return _invoke_function_over_columns("split_part", src, delimiter, partNum) @_try_remote_functions -def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns col1 if it is not NaN, or col2 if col1 is NaN. - - Both inputs should be floating point columns (:class:`DoubleType` or :class:`FloatType`). - - .. versionadded:: 1.6.0 +def substr( + str: "ColumnOrName", pos: "ColumnOrName", len: Optional["ColumnOrName"] = None +) -> Column: + """ + Returns the substring of `str` that starts at `pos` and is of length `len`, + or the slice of byte array that starts at `pos` and is of length `len`. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to check. - A column that evaluates to a double or float. - col2 : :class:`~pyspark.sql.Column` or column name - second column to return if first is NaN. - A column that evaluates to a double or float. + str : :class:`~pyspark.sql.Column` or column name + A column of string. + A column that evaluates to a string or binary. + pos : :class:`~pyspark.sql.Column` or column name + The starting position of the substring. + A column that evaluates to an integer. + len : :class:`~pyspark.sql.Column` or column name, optional + The length of the substring. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - value from first column or second if first is NaN . - Returns a column of the same type as the first input. + substring of given value. + Returns a column of the same type as the input. - Examples + See Also -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) - >>> df.select("*", sf.nanvl("a", "b"), sf.nanvl(df.a, df.b)).show() - +---+---+-----------+-----------+ - | a| b|nanvl(a, b)|nanvl(a, b)| - +---+---+-----------+-----------+ - |1.0|NaN| 1.0| 1.0| - |NaN|2.0| 2.0| 2.0| - +---+---+-----------+-----------+ + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.Column.substr` + :meth:`pyspark.sql.functions.locate` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Spark SQL", 5, 1,)], ["a", "b", "c"]) + >>> df.select("*", sf.substr("a", "b", "c")).show() + +---------+---+---+---------------+ + | a| b| c|substr(a, b, c)| + +---------+---+---+---------------+ + |Spark SQL| 5| 1| k| + +---------+---+---+---------------+ + + >>> df.select("*", sf.substr(df.a, df.b)).show() + +---------+---+---+------------------------+ + | a| b| c|substr(a, b, 2147483647)| + +---------+---+---+------------------------+ + |Spark SQL| 5| 1| k SQL| + +---------+---+---+------------------------+ """ - return _invoke_function_over_columns("nanvl", col1, col2) + if len is not None: + return _invoke_function_over_columns("substr", str, pos, len) + else: + return _invoke_function_over_columns("substr", str, pos) @_try_remote_functions -def percentile( - col: "ColumnOrName", - percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], - frequency: Union[Column, int] = 1, -) -> Column: - """Returns the exact percentile(s) of numeric column `expr` at the given percentage(s) - with value range in [0.0, 1.0]. +def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: + """ + Formats the arguments in printf-style and returns the result as a string column. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats - percentage in decimal (must be between 0.0 and 1.0). - frequency : :class:`~pyspark.sql.Column` or int is a positive numeric literal which - controls frequency. - - Returns - ------- - :class:`~pyspark.sql.Column` - the exact `percentile` of the numeric column. + format : :class:`~pyspark.sql.Column` or str + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or str + column names or :class:`~pyspark.sql.Column`\\s to be used in formatting + Each a column of any type. See Also -------- - :meth:`pyspark.sql.functions.median` - :meth:`pyspark.sql.functions.approx_percentile` - :meth:`pyspark.sql.functions.percentile_approx` + :meth:`pyspark.sql.functions.format_string` Examples -------- - >>> from pyspark.sql import functions as sf - >>> key = (sf.col("id") % 3).alias("key") - >>> value = (sf.randn(42) + key * 10).alias("value") - >>> df = spark.range(0, 1000, 1, 1).select(key, value) - >>> df.select( - ... sf.percentile("value", [0.25, 0.5, 0.75], sf.lit(1)) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |percentile(value, array(0.25, 0.5, 0.75), 1) | - +--------------------------------------------------------+ - |[0.7441991494121..., 9.9900713756..., 19.33740203080...]| - +--------------------------------------------------------+ - - >>> df.groupBy("key").agg( - ... sf.percentile("value", sf.lit(0.5), sf.lit(1)) - ... ).sort("key").show() - +---+-------------------------+ - |key|percentile(value, 0.5, 1)| - +---+-------------------------+ - | 0| -0.03449962216667901| - | 1| 9.990389751837329| - | 2| 19.967859769284075| - +---+-------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("aa%d%s", 123, "cc",)], ["a", "b", "c"] + ... ).select(sf.printf("a", "b", "c")).show() + +---------------+ + |printf(a, b, c)| + +---------------+ + | aa123cc| + +---------------+ """ - percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) - return _invoke_function_over_columns("percentile", col, percentage, lit(frequency)) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function("printf", _to_java_column(format), _to_seq(sc, cols, _to_java_column)) @_try_remote_functions -def percentile_approx( - col: "ColumnOrName", - percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], - accuracy: Union[Column, int] = 10000, +def position( + substr: "ColumnOrName", str: "ColumnOrName", start: Optional["ColumnOrName"] = None ) -> Column: - """Returns the approximate `percentile` of the numeric column `col` which is the smallest value - in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` - of `col` values is less than the value or equal to that value. - - - .. versionadded:: 3.1.0 + """ + Returns the position of the first occurrence of `substr` in `str` after position `start`. + The given `start` and return value are 1-based. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column. - percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats - percentage in decimal (must be between 0.0 and 1.0). - When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. - In this case, returns the approximate percentile array of column col - at the given percentage array. - accuracy : :class:`~pyspark.sql.Column` or int - is a positive numeric literal which controls approximation accuracy - at the cost of memory. Higher value of accuracy yields better accuracy, - 1.0/accuracy is the relative error of the approximation. (default: 10000). - - Returns - ------- - :class:`~pyspark.sql.Column` - approximate `percentile` of the numeric column. - - See Also - -------- - :meth:`pyspark.sql.functions.median` - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.approx_percentile` + substr : :class:`~pyspark.sql.Column` or str + A column of string, substring. + A column that evaluates to a string. + str : :class:`~pyspark.sql.Column` or str + A column of string. + A column that evaluates to a string. + start : :class:`~pyspark.sql.Column` or str, optional + The start position. + A column that evaluates to an integer. Examples -------- - >>> from pyspark.sql import functions as sf - >>> key = (sf.col("id") % 3).alias("key") - >>> value = (sf.randn(42) + key * 10).alias("value") - >>> df = spark.range(0, 1000, 1, 1).select(key, value) - >>> df.select( - ... sf.percentile_approx("value", [0.25, 0.5, 0.75], 1000000) - ... ).show(truncate=False) - +----------------------------------------------------------+ - |percentile_approx(value, array(0.25, 0.5, 0.75), 1000000) | - +----------------------------------------------------------+ - |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| - +----------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] + ... ).select(sf.position("a", "b", "c")).show() + +-----------------+ + |position(a, b, c)| + +-----------------+ + | 7| + +-----------------+ - >>> df.groupBy("key").agg( - ... sf.percentile_approx("value", sf.lit(0.5), sf.lit(1000000)) - ... ).sort("key").show() - +---+--------------------------------------+ - |key|percentile_approx(value, 0.5, 1000000)| - +---+--------------------------------------+ - | 0| -0.03519435193070...| - | 1| 9.990389751837...| - | 2| 19.967859769284...| - +---+--------------------------------------+ + >>> spark.createDataFrame( + ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] + ... ).select(sf.position("a", "b")).show() + +-----------------+ + |position(a, b, 1)| + +-----------------+ + | 4| + +-----------------+ """ - percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) - return _invoke_function_over_columns("percentile_approx", col, percentage, lit(accuracy)) + if start is not None: + return _invoke_function_over_columns("position", substr, str, start) + else: + return _invoke_function_over_columns("position", substr, str) @_try_remote_functions -def approx_percentile( - col: "ColumnOrName", - percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], - accuracy: Union[Column, int] = 10000, -) -> Column: - """Returns the approximate `percentile` of the numeric column `col` which is the smallest value - in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` - of `col` values is less than the value or equal to that value. +def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: + """ + Returns a boolean. The value is True if str ends with suffix. + Returns NULL if either input expression is NULL. Otherwise, returns False. + Both str or suffix must be of STRING or BINARY type. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column. - percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats - percentage in decimal (must be between 0.0 and 1.0). - When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. - In this case, returns the approximate percentile array of column col - at the given percentage array. - accuracy : :class:`~pyspark.sql.Column` or int - is a positive numeric literal which controls approximation accuracy - at the cost of memory. Higher value of accuracy yields better accuracy, - 1.0/accuracy is the relative error of the approximation. (default: 10000). - - Returns - ------- - :class:`~pyspark.sql.Column` - approximate `percentile` of the numeric column. - - See Also - -------- - :meth:`pyspark.sql.functions.median` - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.percentile_approx` + str : :class:`~pyspark.sql.Column` or str + The input value to test. + A column that evaluates to a string or binary. + suffix : :class:`~pyspark.sql.Column` or str + The suffix to test for. + A column that evaluates to a string or binary. Examples -------- - >>> from pyspark.sql import functions as sf - >>> key = (sf.col("id") % 3).alias("key") - >>> value = (sf.randn(42) + key * 10).alias("value") - >>> df = spark.range(0, 1000, 1, 1).select(key, value) - >>> df.select( - ... sf.approx_percentile("value", [0.25, 0.5, 0.75], 1000000) - ... ).show(truncate=False) - +----------------------------------------------------------+ - |approx_percentile(value, array(0.25, 0.5, 0.75), 1000000) | - +----------------------------------------------------------+ - |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| - +----------------------------------------------------------+ + >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) + >>> df.select(endswith(df.a, df.b).alias('r')).collect() + [Row(r=False)] - >>> df.groupBy("key").agg( - ... sf.approx_percentile("value", sf.lit(0.5), sf.lit(1000000)) - ... ).sort("key").show() - +---+--------------------------------------+ - |key|approx_percentile(value, 0.5, 1000000)| - +---+--------------------------------------+ - | 0| -0.03519435193070...| - | 1| 9.990389751837...| - | 2| 19.967859769284...| - +---+--------------------------------------+ + >>> df = spark.createDataFrame([("414243", "4243",)], ["e", "f"]) + >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) + >>> df.printSchema() + root + |-- e: binary (nullable = true) + |-- f: binary (nullable = true) + >>> df.select(endswith("e", "f"), endswith("f", "e")).show() + +--------------+--------------+ + |endswith(e, f)|endswith(f, e)| + +--------------+--------------+ + | true| false| + +--------------+--------------+ """ - percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) - return _invoke_function_over_columns("approx_percentile", col, percentage, lit(accuracy)) + return _invoke_function_over_columns("endswith", str, suffix) @_try_remote_functions -def rand(seed: Optional[int] = None) -> Column: - """Generates a random column with independent and identically distributed (i.i.d.) samples - uniformly distributed in [0.0, 1.0). - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. +def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: + """ + Returns a boolean. The value is True if str starts with prefix. + Returns NULL if either input expression is NULL. Otherwise, returns False. + Both str or prefix must be of STRING or BINARY type. - Notes - ----- - The function is non-deterministic in general case. + .. versionadded:: 3.5.0 Parameters ---------- - seed : int, optional - Seed value for the random generator. - A column that evaluates to an integer or long. Must be a constant. - - Returns - ------- - :class:`~pyspark.sql.Column` - A column of random values. - Returns a column that evaluates to a double. - - See Also - -------- - :meth:`pyspark.sql.functions.randn` - :meth:`pyspark.sql.functions.randstr` - :meth:`pyspark.sql.functions.uniform` + str : :class:`~pyspark.sql.Column` or str + The input value to test. + A column that evaluates to a string or binary. + prefix : :class:`~pyspark.sql.Column` or str + The prefix to test for. + A column that evaluates to a string or binary. Examples -------- - Example 1: Generate a random column without a seed + >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) + >>> df.select(startswith(df.a, df.b).alias('r')).collect() + [Row(r=True)] - >>> from pyspark.sql import functions as sf - >>> spark.range(0, 2, 1, 1).select("*", sf.rand()).show() # doctest: +SKIP - +---+-------------------------+ - | id|rand(-158884697681280011)| - +---+-------------------------+ - | 0| 0.9253464547887...| - | 1| 0.6533254118758...| - +---+-------------------------+ + >>> df = spark.createDataFrame([("414243", "4142",)], ["e", "f"]) + >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) + >>> df.printSchema() + root + |-- e: binary (nullable = true) + |-- f: binary (nullable = true) + >>> df.select(startswith("e", "f"), startswith("f", "e")).show() + +----------------+----------------+ + |startswith(e, f)|startswith(f, e)| + +----------------+----------------+ + | true| false| + +----------------+----------------+ + """ + return _invoke_function_over_columns("startswith", str, prefix) - Example 2: Generate a random column with a specific seed - >>> spark.range(0, 2, 1, 1).select("*", sf.rand(seed=42)).show() - +---+------------------+ - | id| rand(42)| - +---+------------------+ - | 0| 0.619189370225...| - | 1|0.5096018842446...| - +---+------------------+ +@_try_remote_functions +def char(col: "ColumnOrName") -> Column: """ - if seed is not None: - return _invoke_function("rand", _enum_to_value(seed)) - else: - return _invoke_function("rand") - + Returns the ASCII character having the binary equivalent to `col`. If col is larger than 256 the + result is equivalent to char(col % 256) -random = rand + .. versionadded:: 3.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a long. -@_try_remote_functions -def randn(seed: Optional[int] = None) -> Column: - """Generates a random column with independent and identically distributed (i.i.d.) samples - from the standard normal distribution. + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.char(sf.lit(65))).show() + +--------+ + |char(65)| + +--------+ + | A| + +--------+ + """ + return _invoke_function_over_columns("char", col) - .. versionadded:: 1.4.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: + """ + Remove the leading and trailing `trim` characters from `str`. - Notes - ----- - The function is non-deterministic in general case. + .. versionadded:: 3.5.0 Parameters ---------- - seed : int (default: None) - Seed value for the random generator. - A column that evaluates to an integer or long. Must be a constant. - - Returns - ------- - :class:`~pyspark.sql.Column` - A column of random values. - Returns a column that evaluates to a double. - - See Also - -------- - :meth:`pyspark.sql.functions.rand` - :meth:`pyspark.sql.functions.randstr` - :meth:`pyspark.sql.functions.uniform` + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or str, optional + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. Examples -------- - Example 1: Generate a random column without a seed - - >>> from pyspark.sql import functions as sf - >>> spark.range(0, 2, 1, 1).select("*", sf.randn()).show() # doctest: +SKIP - +---+--------------------------+ - | id|randn(3968742514375399317)| - +---+--------------------------+ - | 0| -0.47968645355788...| - | 1| -0.4950952457305...| - +---+--------------------------+ - - Example 2: Generate a random column with a specific seed + >>> df = spark.createDataFrame([("SSparkSQLS", "SL", )], ['a', 'b']) + >>> df.select(btrim(df.a, df.b).alias('r')).collect() + [Row(r='parkSQ')] - >>> spark.range(0, 2, 1, 1).select("*", sf.randn(seed=42)).show() - +---+------------------+ - | id| randn(42)| - +---+------------------+ - | 0| 2.384479054241...| - | 1|0.1920934041293...| - +---+------------------+ + >>> df = spark.createDataFrame([(" SparkSQL ",)], ['a']) + >>> df.select(btrim(df.a).alias('r')).collect() + [Row(r='SparkSQL')] """ - if seed is not None: - return _invoke_function("randn", _enum_to_value(seed)) + if trim is not None: + return _invoke_function_over_columns("btrim", str, trim) else: - return _invoke_function("randn") + return _invoke_function_over_columns("btrim", str) @_try_remote_functions -def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: +def char_length(str: "ColumnOrName") -> Column: """ - Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 - or at integral part when `scale` < 0. - - .. versionadded:: 1.5.0 + Returns the character length of string data or number of bytes of binary data. + The length of string data includes the trailing spaces. + The length of binary data includes binary zeros. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column or column name to compute the round on. - A column that evaluates to a numeric. - scale : :class:`~pyspark.sql.Column` or int, optional - An optional parameter to control the rounding behavior. - A column that evaluates to an integer. Must be a constant. - - .. versionchanged:: 4.0.0 - Support Column type. + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string or binary. - Returns - ------- - :class:`~pyspark.sql.Column` - A column for the rounded value. - Returns a column of the same type as the input. + See Also + -------- + :meth:`pyspark.sql.functions.character_length` + :meth:`pyspark.sql.functions.length` Examples -------- - Example 1: Compute the rounded of a column value - - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.round(sf.lit(2.5))).show() - +-------------+ - |round(2.5, 0)| - +-------------+ - | 3.0| - +-------------+ - - Example 2: Compute the rounded of a column value with a specified scale - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.round(sf.lit(2.1267), sf.lit(2))).show() - +----------------+ - |round(2.1267, 2)| - +----------------+ - | 2.13| - +----------------+ + >>> spark.range(1).select(sf.char_length(sf.lit("SparkSQL"))).show() + +---------------------+ + |char_length(SparkSQL)| + +---------------------+ + | 8| + +---------------------+ """ - if scale is None: - return _invoke_function_over_columns("round", col) - else: - scale = _enum_to_value(scale) - scale = lit(scale) if isinstance(scale, int) else scale - return _invoke_function_over_columns("round", col, scale) + return _invoke_function_over_columns("char_length", str) @_try_remote_functions -def truncate(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: +def character_length(str: "ColumnOrName") -> Column: """ - Truncate the given value toward zero to `scale` decimal places when `scale` >= 0, - or to the left of the decimal point when `scale` < 0. `scale` defaults to 0. - - Unlike :func:`round`, the result is always rounded toward zero, and unlike :func:`floor` - negative values are not rounded toward negative infinity. + Returns the character length of string data or number of bytes of binary data. + The length of string data includes the trailing spaces. + The length of binary data includes binary zeros. - .. versionadded:: 4.4.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column or column name to truncate. - A column that evaluates to a numeric. - scale : :class:`~pyspark.sql.Column` or int, optional - An optional parameter to control the number of decimal places to keep. - A column that evaluates to an integer. Must be a constant. Defaults to 0. - - Returns - ------- - :class:`~pyspark.sql.Column` - A column for the truncated value, of the same type as the input, except that a decimal - input may return a decimal of different precision and scale. + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string or binary. See Also -------- - :meth:`pyspark.sql.functions.round` - :meth:`pyspark.sql.functions.trunc` - :meth:`pyspark.sql.functions.floor` - :meth:`pyspark.sql.functions.ceil` + :meth:`pyspark.sql.functions.char_length` + :meth:`pyspark.sql.functions.length` Examples -------- - Example 1: Truncate toward zero to a given number of decimal places - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.truncate(sf.lit(15.79), sf.lit(1)).alias("r")).collect() - [Row(r=15.7)] + >>> spark.range(1).select(sf.character_length(sf.lit("SparkSQL"))).show() + +--------------------------+ + |character_length(SparkSQL)| + +--------------------------+ + | 8| + +--------------------------+ + """ + return _invoke_function_over_columns("character_length", str) - Example 2: Truncation rounds toward zero for negative values - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.truncate(sf.lit(-2.99), sf.lit(0)).alias("r")).collect() - [Row(r=-2.0)] +@_try_remote_functions +def chr(n: "ColumnOrName") -> Column: + """ + Returns the ASCII character having the binary equivalent to `n`. + If n is larger than 256 the result is equivalent to chr(n % 256). - Example 3: The scale argument defaults to 0 when omitted + .. versionadded:: 4.1.0 + + Parameters + ---------- + n : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a long. + Examples + -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.truncate(sf.lit(1234.5678)).alias("r")).collect() - [Row(r=1234.0)] + >>> spark.range(60, 70).select("*", sf.chr("id")).show() + +---+-------+ + | id|chr(id)| + +---+-------+ + | 60| <| + | 61| =| + | 62| >| + | 63| ?| + | 64| @| + | 65| A| + | 66| B| + | 67| C| + | 68| D| + | 69| E| + +---+-------+ """ - if scale is None: - return _invoke_function_over_columns("truncate", col) - else: - scale = _enum_to_value(scale) - scale = lit(scale) if isinstance(scale, int) else scale - return _invoke_function_over_columns("truncate", col, scale) + return _invoke_function_over_columns("chr", n) @_try_remote_functions -def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: +def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: """ - Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 - or at integral part when `scale` < 0. - - .. versionadded:: 2.0.0 + This is a special version of `to_binary` that performs the same operation, but returns a NULL + value instead of raising an error if the conversion cannot be performed. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column or column name to compute the round on. - A column that evaluates to a numeric. - scale : :class:`~pyspark.sql.Column` or int, optional - An optional parameter to control the rounding behavior. - A column that evaluates to an integer. Must be a constant. - - .. versionchanged:: 4.0.0 - Support Column type. + col : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert binary values. + A column that evaluates to a string. Must be a constant. - Returns - ------- - :class:`~pyspark.sql.Column` - A column for the rounded value. - Returns a column of the same type as the input. + See Also + -------- + :meth:`pyspark.sql.functions.to_binary` Examples -------- - Example 1: Compute the rounded of a column value + Example 1: Convert string to a binary with encoding specified >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.bround(sf.lit(2.5))).show() - +--------------+ - |bround(2.5, 0)| - +--------------+ - | 2.0| - +--------------+ + >>> df = spark.createDataFrame([("abc",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() + [Row(r=b'abc')] - Example 2: Compute the rounded of a column value with a specified scale + Example 2: Convert string to a timestamp without encoding specified >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.bround(sf.lit(2.1267), sf.lit(2))).show() - +-----------------+ - |bround(2.1267, 2)| - +-----------------+ - | 2.13| - +-----------------+ - """ - if scale is None: - return _invoke_function_over_columns("bround", col) - else: - scale = _enum_to_value(scale) - scale = lit(scale) if isinstance(scale, int) else scale - return _invoke_function_over_columns("bround", col, scale) - - -@_try_remote_functions -def shiftLeft(col: "ColumnOrName", numBits: int) -> Column: - """Shift the given value numBits left. - - .. versionadded:: 1.5.0 + >>> df = spark.createDataFrame([("414243",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() + [Row(r=b'ABC')] - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Converion failure results in NULL when ANSI mode is on - .. deprecated:: 3.2.0 - Use :func:`shiftleft` instead. + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.range(1) + ... df.select(sf.try_to_binary(sf.lit("malformed"), sf.lit("hex"))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-----------------------------+ + |try_to_binary(malformed, hex)| + +-----------------------------+ + | NULL| + +-----------------------------+ """ - warnings.warn("Deprecated in 3.2, use shiftleft instead.", FutureWarning) - return shiftleft(col, numBits) + if format is not None: + return _invoke_function_over_columns("try_to_binary", col, format) + else: + return _invoke_function_over_columns("try_to_binary", col) @_try_remote_functions -def shiftleft(col: "ColumnOrName", numBits: int) -> Column: - """Shift the given value numBits left. - - .. versionadded:: 3.2.0 +def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: + """ + Convert string 'col' to a number based on the string format `format`. Returns NULL if the + string 'col' does not match the expected format. The format follows the same semantics as the + to_number function. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to shift. - A column that evaluates to an integer or long. - numBits : int - number of bits to shift. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert number values. + A column that evaluates to a string. Must be a constant. - Returns - ------- - :class:`~pyspark.sql.Column` - shifted value. - Returns a column of the same type as the input. + See Also + -------- + :meth:`pyspark.sql.functions.to_number` Examples -------- + Example 1: Convert a string to a number with a format specified + >>> import pyspark.sql.functions as sf - >>> spark.range(4).select("*", sf.shiftleft('id', 1)).show() - +---+----------------+ - | id|shiftleft(id, 1)| - +---+----------------+ - | 0| 0| - | 1| 2| - | 2| 4| - | 3| 6| - +---+----------------+ - """ - from pyspark.sql.classic.column import _to_java_column + >>> df = spark.createDataFrame([("$78.12",)], ["e"]) + >>> df.select(sf.try_to_number(df.e, sf.lit("$99.99")).alias('r')).show() + +-----+ + | r| + +-----+ + |78.12| + +-----+ - return _invoke_function("shiftleft", _to_java_column(col), _enum_to_value(numBits)) + Example 2: Converion failure results in NULL when ANSI mode is on + + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.range(1) + ... df.select(sf.try_to_number(sf.lit("77"), sf.lit("$99.99")).alias('r')).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +----+ + | r| + +----+ + |NULL| + +----+ + """ + return _invoke_function_over_columns("try_to_number", col, format) @_try_remote_functions -def shiftRight(col: "ColumnOrName", numBits: int) -> Column: - """(Signed) shift the given value numBits right. +def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """ + Returns a boolean. The value is True if right is found inside left. + Returns NULL if either input expression is NULL. Otherwise, returns False. + Both left or right must be of STRING or BINARY type. - .. versionadded:: 1.5.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Parameters + ---------- + left : :class:`~pyspark.sql.Column` or str + The input to check; may be NULL. + A column that evaluates to a string or binary. + right : :class:`~pyspark.sql.Column` or str + The value to find; may be NULL. + A column that evaluates to a string or binary. - .. deprecated:: 3.2.0 - Use :func:`shiftright` instead. + Examples + -------- + >>> df = spark.createDataFrame([("Spark SQL", "Spark")], ['a', 'b']) + >>> df.select(contains(df.a, df.b).alias('r')).collect() + [Row(r=True)] + + >>> df = spark.createDataFrame([("414243", "4243",)], ["c", "d"]) + >>> df = df.select(to_binary("c").alias("c"), to_binary("d").alias("d")) + >>> df.printSchema() + root + |-- c: binary (nullable = true) + |-- d: binary (nullable = true) + >>> df.select(contains("c", "d"), contains("d", "c")).show() + +--------------+--------------+ + |contains(c, d)|contains(d, c)| + +--------------+--------------+ + | true| false| + +--------------+--------------+ """ - warnings.warn("Deprecated in 3.2, use shiftright instead.", FutureWarning) - return shiftright(col, numBits) + return _invoke_function_over_columns("contains", left, right) @_try_remote_functions -def shiftright(col: "ColumnOrName", numBits: int) -> Column: - """(Signed) shift the given value numBits right. - - .. versionadded:: 3.2.0 +def elt(*inputs: "ColumnOrName") -> Column: + """ + Returns the `n`-th input, e.g., returns `input2` when `n` is 2. + The function returns NULL if the index exceeds the length of the array + and `spark.sql.ansi.enabled` is set to false. If `spark.sql.ansi.enabled` is set to true, + it throws ArrayIndexOutOfBoundsException for invalid indices. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to shift. - A column that evaluates to an integer or long. - numBits : int - number of bits to shift. - A column that evaluates to an integer. - - Returns - ------- - :class:`~pyspark.sql.Column` - shifted values. - Returns a column of the same type as the input. + inputs : :class:`~pyspark.sql.Column` or str + Input columns or strings. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(4).select("*", sf.shiftright('id', 1)).show() - +---+-----------------+ - | id|shiftright(id, 1)| - +---+-----------------+ - | 0| 0| - | 1| 0| - | 2| 1| - | 3| 1| - +---+-----------------+ + >>> df = spark.createDataFrame([(1, "scala", "java")], ['a', 'b', 'c']) + >>> df.select(elt(df.a, df.b, df.c).alias('r')).collect() + [Row(r='scala')] """ - from pyspark.sql.classic.column import _to_java_column + from pyspark.sql.classic.column import _to_java_column, _to_seq - return _invoke_function("shiftright", _to_java_column(col), _enum_to_value(numBits)) + sc = _get_active_spark_context() + return _invoke_function("elt", _to_seq(sc, inputs, _to_java_column)) @_try_remote_functions -def shiftRightUnsigned(col: "ColumnOrName", numBits: int) -> Column: - """Unsigned shift the given value numBits right. +def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: + """ + Returns the index (1-based) of the given string (`str`) in the comma-delimited + list (`strArray`). Returns 0, if the string was not found or if the given string (`str`) + contains a comma. - .. versionadded:: 1.5.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + The given string to be found. + A column that evaluates to a string. + str_array : :class:`~pyspark.sql.Column` or str + The comma-delimited list. + A column that evaluates to a string. - .. deprecated:: 3.2.0 - Use :func:`shiftrightunsigned` instead. + Examples + -------- + >>> df = spark.createDataFrame([("ab", "abc,b,ab,c,def")], ['a', 'b']) + >>> df.select(find_in_set(df.a, df.b).alias('r')).collect() + [Row(r=3)] """ - warnings.warn("Deprecated in 3.2, use shiftrightunsigned instead.", FutureWarning) - return shiftrightunsigned(col, numBits) + return _invoke_function_over_columns("find_in_set", str, str_array) @_try_remote_functions -def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: - """Unsigned shift the given value numBits right. - - .. versionadded:: 3.2.0 +def lcase(str: "ColumnOrName") -> Column: + """ + Returns `str` with all characters changed to lowercase. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to shift. - A column that evaluates to an integer or long. - numBits : int - number of bits to shift. - A column that evaluates to an integer. + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. - Returns - ------- - :class:`~pyspark.sql.Column` - shifted value. - Returns a column of the same type as the input. + See Also + -------- + :meth:`pyspark.sql.functions.lower` + :meth:`pyspark.sql.functions.ucase` + :meth:`pyspark.sql.functions.upper` Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(4).select("*", sf.shiftrightunsigned(sf.col('id') - 2, 1)).show() - +---+-------------------------------+ - | id|shiftrightunsigned((id - 2), 1)| - +---+-------------------------------+ - | 0| 9223372036854775807| - | 1| 9223372036854775807| - | 2| 0| - | 3| 0| - +---+-------------------------------+ + >>> spark.range(1).select(sf.lcase(sf.lit("Spark"))).show() + +------------+ + |lcase(Spark)| + +------------+ + | spark| + +------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("shiftrightunsigned", _to_java_column(col), _enum_to_value(numBits)) + return _invoke_function_over_columns("lcase", str) @_try_remote_functions -def spark_partition_id() -> Column: - """A column for partition ID. - - .. versionadded:: 1.6.0 +def ucase(str: "ColumnOrName") -> Column: + """ + Returns `str` with all characters changed to uppercase. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 - Notes - ----- - This is non deterministic because it depends on data partitioning and task scheduling. + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. - Returns - ------- - :class:`~pyspark.sql.Column` - partition id the record belongs to. + See Also + -------- + :meth:`pyspark.sql.functions.upper` + :meth:`pyspark.sql.functions.lcase` + :meth:`pyspark.sql.functions.lower` Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(10, numPartitions=5).select("*", sf.spark_partition_id()).show() - +---+--------------------+ - | id|SPARK_PARTITION_ID()| - +---+--------------------+ - | 0| 0| - | 1| 0| - | 2| 1| - | 3| 1| - | 4| 2| - | 5| 2| - | 6| 3| - | 7| 3| - | 8| 4| - | 9| 4| - +---+--------------------+ + >>> spark.range(1).select(sf.ucase(sf.lit("Spark"))).show() + +------------+ + |ucase(Spark)| + +------------+ + | SPARK| + +------------+ """ - return _invoke_function("spark_partition_id") + return _invoke_function_over_columns("ucase", str) @_try_remote_functions -def expr(str: str) -> Column: - """Parses the expression string into the column that it represents - - .. versionadded:: 1.5.0 +def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: + """ + Returns the leftmost `len`(`len` can be string type) characters from the string `str`, + if `len` is less or equal than 0 the result is an empty string. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - str : expression string - expression defined in string. - - Returns - ------- - :class:`~pyspark.sql.Column` - column representing the expression. + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string or binary. + len : :class:`~pyspark.sql.Column` or str + Input column or strings, the leftmost `len`. + A column that evaluates to an integer. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([["Alice"], ["Bob"]], ["name"]) - >>> df.select("*", sf.expr("length(name)")).show() - +-----+------------+ - | name|length(name)| - +-----+------------+ - |Alice| 5| - | Bob| 3| - +-----+------------+ + >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) + >>> df.select(left(df.a, df.b).alias('r')).collect() + [Row(r='Spa')] """ - return _invoke_function("expr", str) - - -@overload -def struct(*cols: "ColumnOrName") -> Column: ... + return _invoke_function_over_columns("left", str, len) -@overload -def struct(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... +@_try_remote_functions +def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: + """ + Returns the rightmost `len`(`len` can be string type) characters from the string `str`, + if `len` is less or equal than 0 the result is an empty string. - -@_try_remote_functions -def struct( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], -) -> Column: - """Creates a new struct column. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - cols : list, set, :class:`~pyspark.sql.Column` or column name - column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct. - Each a column of any type. - - Returns - ------- - :class:`~pyspark.sql.Column` - a struct type column of given columns. - Returns a column that evaluates to a struct. - - See Also - -------- - :meth:`pyspark.sql.functions.named_struct` + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + len : :class:`~pyspark.sql.Column` or str + Input column or strings, the rightmost `len`. + A column that evaluates to an integer. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) - >>> df.select("*", sf.struct('age', df.name)).show() - +-----+---+-----------------+ - | name|age|struct(age, name)| - +-----+---+-----------------+ - |Alice| 2| {2, Alice}| - | Bob| 5| {5, Bob}| - +-----+---+-----------------+ + >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) + >>> df.select(right(df.a, df.b).alias('r')).collect() + [Row(r='SQL')] """ - if len(cols) == 1 and isinstance(cols[0], (list, set)): - cols = cols[0] # type: ignore[assignment] - return _invoke_function_over_seq_of_columns("struct", cols) # type: ignore[arg-type] + return _invoke_function_over_columns("right", str, len) @_try_remote_functions -def named_struct(*cols: "ColumnOrName") -> Column: +def mask( + col: "ColumnOrName", + upperChar: Optional["ColumnOrName"] = None, + lowerChar: Optional["ColumnOrName"] = None, + digitChar: Optional["ColumnOrName"] = None, + otherChar: Optional["ColumnOrName"] = None, +) -> Column: """ - Creates a struct with the given field names and values. + Masks the given string value. This can be useful for creating copies of tables with sensitive + information removed. .. versionadded:: 3.5.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - list of columns to work on. + col: :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to a string. + upperChar: :class:`~pyspark.sql.Column` or str, optional + character to replace upper-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. + lowerChar: :class:`~pyspark.sql.Column` or str, optional + character to replace lower-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. + digitChar: :class:`~pyspark.sql.Column` or str, optional + character to replace digit characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. + otherChar: :class:`~pyspark.sql.Column` or str, optional + character to replace all other characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.struct` + Returns a column that evaluates to a string. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, 2)], ['a', 'b']) - >>> df.select("*", sf.named_struct(sf.lit('x'), df.a, sf.lit('y'), "b")).show() - +---+---+------------------------+ - | a| b|named_struct(x, a, y, b)| - +---+---+------------------------+ - | 1| 2| {1, 2}| - +---+---+------------------------+ + >>> df = spark.createDataFrame([("AbCD123-@$#",), ("abcd-EFGH-8765-4321",)], ['data']) + >>> df.select(mask(df.data).alias('r')).collect() + [Row(r='XxXXnnn-@$#'), Row(r='xxxx-XXXX-nnnn-nnnn')] + >>> df.select(mask(df.data, lit('Y')).alias('r')).collect() + [Row(r='YxYYnnn-@$#'), Row(r='xxxx-YYYY-nnnn-nnnn')] + >>> df.select(mask(df.data, lit('Y'), lit('y')).alias('r')).collect() + [Row(r='YyYYnnn-@$#'), Row(r='yyyy-YYYY-nnnn-nnnn')] + >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d')).alias('r')).collect() + [Row(r='YyYYddd-@$#'), Row(r='yyyy-YYYY-dddd-dddd')] + >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d'), lit('*')).alias('r')).collect() + [Row(r='YyYYddd****'), Row(r='yyyy*YYYY*dddd*dddd')] """ - return _invoke_function_over_seq_of_columns("named_struct", cols) + + _upperChar = lit("X") if upperChar is None else upperChar + _lowerChar = lit("x") if lowerChar is None else lowerChar + _digitChar = lit("n") if digitChar is None else digitChar + _otherChar = lit(None) if otherChar is None else otherChar + return _invoke_function_over_columns( + "mask", col, _upperChar, _lowerChar, _digitChar, _otherChar + ) @_try_remote_functions -def greatest(*cols: "ColumnOrName") -> Column: +def collate(col: "ColumnOrName", collation: str) -> Column: """ - Returns the greatest value of the list of column names, skipping null values. - This function takes at least 2 parameters. It will return null if all parameters are null. - - .. versionadded:: 1.5.0 + Marks a given column with specified collation. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - cols: :class:`~pyspark.sql.Column` or column name - columns to check for greatest value. - Each a column of any orderable type. + col : :class:`~pyspark.sql.Column` or str + Target string column to work on. + collation : str + Target collation name. Returns ------- :class:`~pyspark.sql.Column` - greatest value. - Returns a column of the same type as the input. - - See Also - -------- - :meth:`pyspark.sql.functions.least` - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) - >>> df.select("*", sf.greatest(df.a, "b", df.c)).show() - +---+---+---+-----------------+ - | a| b| c|greatest(a, b, c)| - +---+---+---+-----------------+ - | 1| 4| 3| 4| - +---+---+---+-----------------+ + A new column of string type, where each value has the specified collation. """ - if len(cols) < 2: - raise PySparkValueError( - errorClass="WRONG_NUM_COLUMNS", - messageParameters={"func_name": "greatest", "num_cols": "2"}, - ) - return _invoke_function_over_seq_of_columns("greatest", cols) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("collate", _to_java_column(col), _enum_to_value(collation)) @_try_remote_functions -def least(*cols: "ColumnOrName") -> Column: +def collation(col: "ColumnOrName") -> Column: """ - Returns the least value of the list of column names, skipping null values. - This function takes at least 2 parameters. It will return null if all parameters are null. - - .. versionadded:: 1.5.0 + Returns the collation name of a given column. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - column names or columns to be compared - Each a column of any orderable type. + col : :class:`~pyspark.sql.Column` or str + Target string column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - least value. - Returns a column of the same type as the input. - - See Also - -------- - :meth:`pyspark.sql.functions.greatest` + collation name of a given expression. + Returns a column that evaluates to a string. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) - >>> df.select("*", sf.least(df.a, "b", df.c)).show() - +---+---+---+--------------+ - | a| b| c|least(a, b, c)| - +---+---+---+--------------+ - | 1| 4| 3| 1| - +---+---+---+--------------+ + >>> df = spark.createDataFrame([('name',)], ['dt']) + >>> df.select(collation('dt').alias('collation')).show(truncate=False) + +--------------------------+ + |collation | + +--------------------------+ + |SYSTEM.BUILTIN.UTF8_BINARY| + +--------------------------+ """ - if len(cols) < 2: - raise PySparkValueError( - errorClass="WRONG_NUM_COLUMNS", - messageParameters={"func_name": "least", "num_cols": "2"}, - ) - return _invoke_function_over_seq_of_columns("least", cols) + return _invoke_function_over_columns("collation", col) @_try_remote_functions -def when(condition: Column, value: Any) -> Column: - """Evaluates a list of conditions and returns one of multiple possible result expressions. - If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched - conditions. - - .. versionadded:: 1.4.0 +def quote(col: "ColumnOrName") -> Column: + r"""Returns `str` enclosed by single quotes and each instance of + single quote in it is preceded by a backslash. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- - condition : :class:`~pyspark.sql.Column` - a boolean :class:`~pyspark.sql.Column` expression. - A column that evaluates to a boolean. - value : - a literal value, or a :class:`~pyspark.sql.Column` expression. - A column of any type. + col : :class:`~pyspark.sql.Column` or column name + target column to be quoted. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - column representing when expression. - Returns a column of the same type as the input. - - See Also - -------- - :meth:`pyspark.sql.Column.when` - :meth:`pyspark.sql.Column.otherwise` + quoted string + Returns a column that evaluates to a string. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.range(3) - >>> df.select("*", sf.when(df['id'] == 2, 3).otherwise(4)).show() - +---+------------------------------------+ - | id|CASE WHEN (id = 2) THEN 3 ELSE 4 END| - +---+------------------------------------+ - | 0| 4| - | 1| 4| - | 2| 3| - +---+------------------------------------+ - - >>> df.select("*", sf.when(df.id == 2, df.id + 1)).show() - +---+------------------------------------+ - | id|CASE WHEN (id = 2) THEN (id + 1) END| - +---+------------------------------------+ - | 0| NULL| - | 1| NULL| - | 2| 3| - +---+------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Don't"], "STRING") + >>> df.select("*", sf.quote("value")).show() + +-----+------------+ + |value|quote(value)| + +-----+------------+ + |Don't| 'Don\'t'| + +-----+------------+ """ - # Explicitly not using ColumnOrName type here to make reading condition less opaque - if not isinstance(condition, Column): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column", - "arg_name": "condition", - "arg_type": type(condition).__name__, - }, - ) - value = _enum_to_value(value) - v = value._jc if isinstance(value, Column) else _enum_to_value(value) + return _invoke_function_over_columns("quote", col) - return _invoke_function("when", condition._jc, v) +# ---------------------- Bitwise Functions ---------------------- -@overload -def log(arg1: "ColumnOrName") -> Column: ... +@_try_remote_functions +def bitwiseNOT(col: "ColumnOrName") -> Column: + """ + Computes bitwise not. -@overload -def log(arg1: float, arg2: "ColumnOrName") -> Column: ... + .. versionadded:: 1.4.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. -@_try_remote_functions -def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = None) -> Column: - """Returns the first argument-based logarithm of the second argument. + .. deprecated:: 3.2.0 + Use :func:`bitwise_not` instead. + """ + warnings.warn("Deprecated in 3.2, use bitwise_not instead.", FutureWarning) + return bitwise_not(col) - If there is only one argument, then this takes the natural logarithm of the argument. - .. versionadded:: 1.5.0 +@_try_remote_functions +def bitwise_not(col: "ColumnOrName") -> Column: + """ + Computes bitwise not. + + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - arg1 : :class:`~pyspark.sql.Column`, str or float - base number or actual number (in this case base is `e`). - A column that evaluates to a double. - arg2 : :class:`~pyspark.sql.Column`, str or float, optional - number to calculate logariphm for. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. Returns ------- :class:`~pyspark.sql.Column` - logariphm of given value. - Returns a column that evaluates to a double. - - See Also - -------- - :meth:`pyspark.sql.functions.ln` + the column for computed results. Examples -------- - Example 1: Specify both base number and the input value - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") - >>> df.select("*", sf.log(2.0, df.value)).show() - +-----+---------------+ - |value|LOG(2.0, value)| - +-----+---------------+ - | 1| 0.0| - | 2| 1.0| - | 4| 2.0| - +-----+---------------+ - - Example 2: Return NULL for invalid input values - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (0), (-1), (NULL) AS t(value)") - >>> df.select("*", sf.log(3.0, df.value)).show() - +-----+------------------+ - |value| LOG(3.0, value)| - +-----+------------------+ - | 1| 0.0| - | 2|0.6309297535714...| - | 0| NULL| - | -1| NULL| - | NULL| NULL| - +-----+------------------+ - - Example 3: Specify only the input value (Natural logarithm) - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") - >>> df.select("*", sf.log(df.value)).show() - +-----+------------------+ - |value| ln(value)| - +-----+------------------+ - | 1| 0.0| - | 2|0.6931471805599...| - | 4|1.3862943611198...| - +-----+------------------+ + >>> spark.sql( + ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" + ... ).select("*", sf.bitwise_not("value")).show() + +-----+------+ + |value|~value| + +-----+------+ + | 0| -1| + | 1| -2| + | 2| -3| + | 3| -4| + | NULL| NULL| + +-----+------+ """ - from pyspark.sql.classic.column import _to_java_column - - if arg2 is None: - return _invoke_function_over_columns("log", cast("ColumnOrName", arg1)) - else: - return _invoke_function("log", _enum_to_value(arg1), _to_java_column(arg2)) + return _invoke_function_over_columns("bitwise_not", col) @_try_remote_functions -def ln(col: "ColumnOrName") -> Column: - """Returns the natural logarithm of the argument. +def bit_count(col: "ColumnOrName") -> Column: + """ + Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer, + or NULL if the argument is NULL. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column to calculate logariphm for. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral or boolean. Returns ------- :class:`~pyspark.sql.Column` - natural logarithm of given value. - Returns a column that evaluates to a double. + the number of bits that are set in the argument expr as an unsigned 64-bit integer, + or NULL if the argument is NULL. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.log` + :meth:`pyspark.sql.functions.bit_get` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(10).select("*", sf.ln('id')).show() - +---+------------------+ - | id| ln(id)| - +---+------------------+ - | 0| NULL| - | 1| 0.0| - | 2|0.6931471805599...| - | 3|1.0986122886681...| - | 4|1.3862943611198...| - | 5|1.6094379124341...| - | 6| 1.791759469228...| - | 7|1.9459101490553...| - | 8|2.0794415416798...| - | 9|2.1972245773362...| - +---+------------------+ + >>> spark.sql( + ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" + ... ).select("*", sf.bit_count("value")).show() + +-----+----------------+ + |value|bit_count(value)| + +-----+----------------+ + | 0| 0| + | 1| 1| + | 2| 1| + | 3| 2| + | NULL| NULL| + +-----+----------------+ """ - return _invoke_function_over_columns("ln", col) + return _invoke_function_over_columns("bit_count", col) @_try_remote_functions -def log2(col: "ColumnOrName") -> Column: - """Returns the base-2 logarithm of the argument. - - .. versionadded:: 1.5.0 +def bit_get(col: "ColumnOrName", pos: "ColumnOrName") -> Column: + """ + Returns the value of the bit (0 or 1) at the specified position. + The positions are numbered from right to left, starting at zero. + The position argument cannot be negative. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column to calculate logariphm for. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. + pos : :class:`~pyspark.sql.Column` or column name + The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - logariphm of given value. - Returns a column that evaluates to a double. + the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. + + See Also + -------- + :meth:`pyspark.sql.functions.bit_count` + :meth:`pyspark.sql.functions.getbit` Examples -------- + Example 1: Get the bit with a literal position + >>> from pyspark.sql import functions as sf - >>> spark.range(10).select("*", sf.log2('id')).show() - +---+------------------+ - | id| LOG2(id)| - +---+------------------+ - | 0| NULL| - | 1| 0.0| - | 2| 1.0| - | 3| 1.584962500721...| - | 4| 2.0| - | 5| 2.321928094887...| - | 6| 2.584962500721...| - | 7| 2.807354922057...| - | 8| 3.0| - | 9|3.1699250014423...| - +---+------------------+ + >>> df = spark.createDataFrame([[1],[2],[3],[None]], ["value"]) + >>> df.select("*", sf.bit_get("value", sf.lit(1))).show() + +-----+-----------------+ + |value|bit_get(value, 1)| + +-----+-----------------+ + | 1| 0| + | 2| 1| + | 3| 1| + | NULL| NULL| + +-----+-----------------+ + + Example 2: Get the bit with a column position + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) + >>> df.select("*", sf.bit_get(df.value, "pos")).show() + +-----+----+-------------------+ + |value| pos|bit_get(value, pos)| + +-----+----+-------------------+ + | 1| 2| 0| + | 2| 1| 1| + | 3|NULL| NULL| + | NULL| 1| NULL| + +-----+----+-------------------+ """ - return _invoke_function_over_columns("log2", col) + return _invoke_function_over_columns("bit_get", col, pos) @_try_remote_functions -def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column: +def getbit(col: "ColumnOrName", pos: "ColumnOrName") -> Column: """ - Convert a number in a string column from one base to another. - - .. versionadded:: 1.5.0 + Returns the value of the bit (0 or 1) at the specified position. + The positions are numbered from right to left, starting at zero. + The position argument cannot be negative. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column to convert base for. - A column that evaluates to a string. - fromBase: int - from base number. - A column that evaluates to an integer. - toBase: int - to base number. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. + pos : :class:`~pyspark.sql.Column` or column name + The positions are numbered from right to left, starting at zero. A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - logariphm of given value. - Returns a column that evaluates to a string. + the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. + + See Also + -------- + :meth:`pyspark.sql.functions.bit_get` + :meth:`pyspark.sql.functions.bit_count` Examples -------- + Example 1: Get the bit with a literal position + + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[1], [2], [3], [None]], ["value"] + ... ).select("*", sf.getbit("value", sf.lit(1))).show() + +-----+----------------+ + |value|getbit(value, 1)| + +-----+----------------+ + | 1| 0| + | 2| 1| + | 3| 1| + | NULL| NULL| + +-----+----------------+ + + Example 2: Get the bit with a column position + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("010101",), ( "101",), ("001",)], ['n']) - >>> df.select("*", sf.conv(df.n, 2, 16)).show() - +------+--------------+ - | n|conv(n, 2, 16)| - +------+--------------+ - |010101| 15| - | 101| 5| - | 001| 1| - +------+--------------+ + >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) + >>> df.select("*", sf.getbit(df.value, "pos")).show() + +-----+----+------------------+ + |value| pos|getbit(value, pos)| + +-----+----+------------------+ + | 1| 2| 0| + | 2| 1| 1| + | 3|NULL| NULL| + | NULL| 1| NULL| + +-----+----+------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "conv", _to_java_column(col), _enum_to_value(fromBase), _enum_to_value(toBase) - ) + return _invoke_function_over_columns("getbit", col, pos) @_try_remote_functions -def factorial(col: "ColumnOrName") -> Column: - """ - Computes the factorial of the given value. +def shiftLeft(col: "ColumnOrName", numBits: int) -> Column: + """Shift the given value numBits left. .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - a column to calculate factorial for. - A column that evaluates to an integer. - - Returns - ------- - :class:`~pyspark.sql.Column` - factorial of given value. - Returns a column that evaluates to a long. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> spark.range(10).select("*", sf.factorial('id')).show() - +---+-------------+ - | id|factorial(id)| - +---+-------------+ - | 0| 1| - | 1| 1| - | 2| 2| - | 3| 6| - | 4| 24| - | 5| 120| - | 6| 720| - | 7| 5040| - | 8| 40320| - | 9| 362880| - +---+-------------+ + .. deprecated:: 3.2.0 + Use :func:`shiftleft` instead. """ - return _invoke_function_over_columns("factorial", col) - - -# --------------- Window functions ------------------------ + warnings.warn("Deprecated in 3.2, use shiftleft instead.", FutureWarning) + return shiftleft(col, numBits) @_try_remote_functions -def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: - """ - Window function: returns the value that is `offset` rows before the current row, and - `default` if there is less than `offset` rows before the current row. For example, - an `offset` of one will return the previous row at any given point in the window partition. - - This is equivalent to the LAG function in SQL. +def shiftleft(col: "ColumnOrName", numBits: int) -> Column: + """Shift the given value numBits left. - .. versionadded:: 1.4.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -9236,89 +9084,57 @@ def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - name of column or expression - offset : int, optional default 1 - number of row to extend - default : optional - default value + input column of values to shift. + A column that evaluates to an integer or long. + numBits : int + number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - value before current row based on `offset`. - - See Also - -------- - :meth:`pyspark.sql.functions.lead` + shifted value. + Returns a column of the same type as the input. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ - - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("previous_value", sf.lag("c2").over(w)).show() - +---+---+--------------+ - | c1| c2|previous_value| - +---+---+--------------+ - | a| 1| NULL| - | a| 2| 1| - | a| 3| 2| - | b| 2| NULL| - | b| 8| 2| - +---+---+--------------+ - - >>> df.withColumn("previous_value", sf.lag("c2", 1, 0).over(w)).show() - +---+---+--------------+ - | c1| c2|previous_value| - +---+---+--------------+ - | a| 1| 0| - | a| 2| 1| - | a| 3| 2| - | b| 2| 0| - | b| 8| 2| - +---+---+--------------+ - - >>> df.withColumn("previous_value", sf.lag("c2", 2, -1).over(w)).show() - +---+---+--------------+ - | c1| c2|previous_value| - +---+---+--------------+ - | a| 1| -1| - | a| 2| -1| - | a| 3| 1| - | b| 2| -1| - | b| 8| -1| - +---+---+--------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(4).select("*", sf.shiftleft('id', 1)).show() + +---+----------------+ + | id|shiftleft(id, 1)| + +---+----------------+ + | 0| 0| + | 1| 2| + | 2| 4| + | 3| 6| + +---+----------------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function( - "lag", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) - ) + return _invoke_function("shiftleft", _to_java_column(col), _enum_to_value(numBits)) @_try_remote_functions -def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: +def shiftRight(col: "ColumnOrName", numBits: int) -> Column: + """(Signed) shift the given value numBits right. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 3.2.0 + Use :func:`shiftright` instead. """ - Window function: returns the value that is `offset` rows after the current row, and - `default` if there is less than `offset` rows after the current row. For example, - an `offset` of one will return the next row at any given point in the window partition. + warnings.warn("Deprecated in 3.2, use shiftright instead.", FutureWarning) + return shiftright(col, numBits) - This is equivalent to the LEAD function in SQL. - .. versionadded:: 1.4.0 +@_try_remote_functions +def shiftright(col: "ColumnOrName", numBits: int) -> Column: + """(Signed) shift the given value numBits right. + + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -9326,91 +9142,57 @@ def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - name of column or expression - offset : int, optional default 1 - number of row to extend - default : optional - default value + input column of values to shift. + A column that evaluates to an integer or long. + numBits : int + number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - value after current row based on `offset`. - - See Also - -------- - :meth:`pyspark.sql.functions.lag` + shifted values. + Returns a column of the same type as the input. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ - - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("next_value", sf.lead("c2").over(w)).show() - +---+---+----------+ - | c1| c2|next_value| - +---+---+----------+ - | a| 1| 2| - | a| 2| 3| - | a| 3| NULL| - | b| 2| 8| - | b| 8| NULL| - +---+---+----------+ - - >>> df.withColumn("next_value", sf.lead("c2", 1, 0).over(w)).show() - +---+---+----------+ - | c1| c2|next_value| - +---+---+----------+ - | a| 1| 2| - | a| 2| 3| - | a| 3| 0| - | b| 2| 8| - | b| 8| 0| - +---+---+----------+ - - >>> df.withColumn("next_value", sf.lead("c2", 2, -1).over(w)).show() - +---+---+----------+ - | c1| c2|next_value| - +---+---+----------+ - | a| 1| 3| - | a| 2| -1| - | a| 3| -1| - | b| 2| -1| - | b| 8| -1| - +---+---+----------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(4).select("*", sf.shiftright('id', 1)).show() + +---+-----------------+ + | id|shiftright(id, 1)| + +---+-----------------+ + | 0| 0| + | 1| 0| + | 2| 1| + | 3| 1| + +---+-----------------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function( - "lead", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) - ) + return _invoke_function("shiftright", _to_java_column(col), _enum_to_value(numBits)) @_try_remote_functions -def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = False) -> Column: +def shiftRightUnsigned(col: "ColumnOrName", numBits: int) -> Column: + """Unsigned shift the given value numBits right. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 3.2.0 + Use :func:`shiftrightunsigned` instead. """ - Window function: returns the value that is the `offset`\\th row of the window frame - (counting from 1), and `null` if the size of window frame is less than `offset` rows. + warnings.warn("Deprecated in 3.2, use shiftrightunsigned instead.", FutureWarning) + return shiftrightunsigned(col, numBits) - It will return the `offset`\\th non-null value it sees when `ignoreNulls` is set to - true. If all values are null, then null is returned. - This is equivalent to the nth_value function in SQL. +@_try_remote_functions +def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: + """Unsigned shift the given value numBits right. - .. versionadded:: 3.1.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -9418,490 +9200,119 @@ def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = Fa Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - name of column or expression - offset : int - number of row to use as the value - ignoreNulls : bool, optional - indicates the Nth value should skip null in the - determination of which row to use + input column of values to shift. + A column that evaluates to an integer or long. + numBits : int + number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - value of nth row. - - See Also - -------- - :meth:`pyspark.sql.functions.first_value` - :meth:`pyspark.sql.functions.last_value` + shifted value. + Returns a column of the same type as the input. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ - - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("nth_value", sf.nth_value("c2", 1).over(w)).show() - +---+---+---------+ - | c1| c2|nth_value| - +---+---+---------+ - | a| 1| 1| - | a| 2| 1| - | a| 3| 1| - | b| 2| 2| - | b| 8| 2| - +---+---+---------+ - - >>> df.withColumn("nth_value", sf.nth_value("c2", 2).over(w)).show() - +---+---+---------+ - | c1| c2|nth_value| - +---+---+---------+ - | a| 1| NULL| - | a| 2| 2| - | a| 3| 2| - | b| 2| NULL| - | b| 8| 8| - +---+---+---------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(4).select("*", sf.shiftrightunsigned(sf.col('id') - 2, 1)).show() + +---+-------------------------------+ + | id|shiftrightunsigned((id - 2), 1)| + +---+-------------------------------+ + | 0| 9223372036854775807| + | 1| 9223372036854775807| + | 2| 0| + | 3| 0| + +---+-------------------------------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function( - "nth_value", _to_java_column(col), _enum_to_value(offset), _enum_to_value(ignoreNulls) - ) + return _invoke_function("shiftrightunsigned", _to_java_column(col), _enum_to_value(numBits)) + + +# ---------------------- Date and Timestamp Functions ---------------------- @_try_remote_functions -def any_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: - """Returns some value of `col` for a group of rows. +def curdate() -> Column: + """ + Returns the current date at the start of query evaluation as a :class:`DateType` column. + All calls of current_date within the same query return the same value. .. versionadded:: 3.5.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column of any type. - ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional - if first value is null then look for first non-null value. - A column that evaluates to a boolean. Must be a constant. - Returns ------- :class:`~pyspark.sql.Column` - some value of `col` for a group of rows. + current date. - Examples + See Also -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.select(sf.any_value('c1'), sf.any_value('c2')).show() - +-------------+-------------+ - |any_value(c1)|any_value(c2)| - +-------------+-------------+ - | NULL| 1| - +-------------+-------------+ + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.current_timestamp` + :meth:`pyspark.sql.functions.localtimestamp` - >>> df.select(sf.any_value('c1', True), sf.any_value('c2', True)).show() - +-------------+-------------+ - |any_value(c1)|any_value(c2)| - +-------------+-------------+ - | a| 1| - +-------------+-------------+ + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.curdate()).show() # doctest: +SKIP + +--------------+ + |current_date()| + +--------------+ + | 2022-08-26| + +--------------+ """ - if ignoreNulls is None: - return _invoke_function_over_columns("any_value", col) - else: - ignoreNulls = _enum_to_value(ignoreNulls) - ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls - return _invoke_function_over_columns("any_value", col, ignoreNulls) + return _invoke_function("curdate") @_try_remote_functions -def first_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: - """Returns the first value of `col` for a group of rows. It will return the first non-null - value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. +def current_date() -> Column: + """ + Returns the current date at the start of query evaluation as a :class:`DateType` column. + All calls of current_date within the same query return the same value. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column of any type. - ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional - if first value is null then look for first non-null value. - A column that evaluates to a boolean. Must be a constant. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Returns ------- :class:`~pyspark.sql.Column` - some value of `col` for a group of rows. + current date. See Also -------- - :meth:`pyspark.sql.functions.last_value` - :meth:`pyspark.sql.functions.nth_value` + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.curdate` + :meth:`pyspark.sql.functions.current_timestamp` + :meth:`pyspark.sql.functions.localtimestamp` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] - ... ).select(sf.first_value('a'), sf.first_value('b')).show() - +--------------+--------------+ - |first_value(a)|first_value(b)| - +--------------+--------------+ - | NULL| 1| - +--------------+--------------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] - ... ).select(sf.first_value('a', True), sf.first_value('b', True)).show() - +--------------+--------------+ - |first_value(a)|first_value(b)| - +--------------+--------------+ - | a| 1| - +--------------+--------------+ + >>> from pyspark.sql import functions as sf + >>> spark.range(1).select(sf.current_date()).show() # doctest: +SKIP + +--------------+ + |current_date()| + +--------------+ + | 2022-08-26| + +--------------+ """ - if ignoreNulls is None: - return _invoke_function_over_columns("first_value", col) - else: - ignoreNulls = _enum_to_value(ignoreNulls) - ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls - return _invoke_function_over_columns("first_value", col, ignoreNulls) + return _invoke_function("current_date") @_try_remote_functions -def last_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: - """Returns the last value of `col` for a group of rows. It will return the last non-null - value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. +def current_timezone() -> Column: + """ + Returns the current session local timezone. .. versionadded:: 3.5.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column of any type. - ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional - if first value is null then look for first non-null value. - A column that evaluates to a boolean. Must be a constant. - Returns ------- :class:`~pyspark.sql.Column` - some value of `col` for a group of rows. - - See Also - -------- - :meth:`pyspark.sql.functions.first_value` - :meth:`pyspark.sql.functions.nth_value` - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] - ... ).select(sf.last_value('a'), sf.last_value('b')).show() - +-------------+-------------+ - |last_value(a)|last_value(b)| - +-------------+-------------+ - | NULL| 2| - +-------------+-------------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] - ... ).select(sf.last_value('a', True), sf.last_value('b', True)).show() - +-------------+-------------+ - |last_value(a)|last_value(b)| - +-------------+-------------+ - | b| 2| - +-------------+-------------+ - """ - if ignoreNulls is None: - return _invoke_function_over_columns("last_value", col) - else: - ignoreNulls = _enum_to_value(ignoreNulls) - ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls - return _invoke_function_over_columns("last_value", col, ignoreNulls) - - -@_try_remote_functions -def count_if(col: "ColumnOrName") -> Column: - """ - Aggregate function: Returns the number of `TRUE` values for the `col`. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a boolean. - - Returns - ------- - :class:`~pyspark.sql.Column` - the number of `TRUE` values for the `col`. - - See Also - -------- - :meth:`pyspark.sql.functions.count` - - Examples - -------- - Example 1: Counting the number of even numbers in a numeric column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.select(sf.count_if(sf.col('c2') % 2 == 0)).show() - +------------------------+ - |count_if(((c2 % 2) = 0))| - +------------------------+ - | 3| - +------------------------+ - - Example 2: Counting the number of rows where a string column starts with a certain letter - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("apple",), ("banana",), ("cherry",), ("apple",), ("banana",)], ["fruit"]) - >>> df.select(sf.count_if(sf.col('fruit').startswith('a'))).show() - +------------------------------+ - |count_if(startswith(fruit, a))| - +------------------------------+ - | 2| - +------------------------------+ - - Example 3: Counting the number of rows where a numeric column is greater than a certain value - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,), (5,)], ["num"]) - >>> df.select(sf.count_if(sf.col('num') > 3)).show() - +-------------------+ - |count_if((num > 3))| - +-------------------+ - | 2| - +-------------------+ - - Example 4: Counting the number of rows where a boolean column is True - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(True,), (False,), (True,), (False,), (True,)], ["b"]) - >>> df.select(sf.count('b'), sf.count_if('b')).show() - +--------+-----------+ - |count(b)|count_if(b)| - +--------+-----------+ - | 5| 3| - +--------+-----------+ - """ - return _invoke_function_over_columns("count_if", col) - - -@_try_remote_functions -def histogram_numeric(col: "ColumnOrName", nBins: Column) -> Column: - """Computes a histogram on numeric 'col' using nb bins. - The return value is an array of (x,y) pairs representing the centers of the - histogram's bins. As the value of 'nb' is increased, the histogram approximation - gets finer-grained, but may yield artifacts around outliers. In practice, 20-40 - histogram bins appear to work well, with more bins being required for skewed or - smaller datasets. Note that this function creates a histogram with non-uniform - bin widths. It offers no guarantees in terms of the mean-squared-error of the - histogram, but in practice is comparable to the histograms produced by the R/S-Plus - statistical computing packages. Note: the output type of the 'x' field in the return value is - propagated from the input value consumed in the aggregate function. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - nBins : :class:`~pyspark.sql.Column` - number of Histogram columns. - - Returns - ------- - :class:`~pyspark.sql.Column` - a histogram on numeric 'col' using nb bins. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.range(100, numPartitions=1) - >>> df.select(sf.histogram_numeric('id', sf.lit(5))).show(truncate=False) - +-----------------------------------------------------------+ - |histogram_numeric(id, 5) | - +-----------------------------------------------------------+ - |[{11, 25.0}, {36, 24.0}, {59, 23.0}, {84, 25.0}, {98, 3.0}]| - +-----------------------------------------------------------+ - """ - return _invoke_function_over_columns("histogram_numeric", col, nBins) - - -@_try_remote_functions -def ntile(n: int) -> Column: - """ - Window function: returns the ntile group id (from 1 to `n` inclusive) - in an ordered window partition. For example, if `n` is 4, the first - quarter of the rows will get value 1, the second quarter will get 2, - the third quarter will get 3, and the last quarter will get 4. - - This is equivalent to the NTILE function in SQL. - - .. versionadded:: 1.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - n : int - an integer - - Returns - ------- - :class:`~pyspark.sql.Column` - portioned group id. - - See Also - -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ - - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("ntile", sf.ntile(2).over(w)).show() - +---+---+-----+ - | c1| c2|ntile| - +---+---+-----+ - | a| 1| 1| - | a| 2| 1| - | a| 3| 2| - | b| 2| 1| - | b| 8| 2| - +---+---+-----+ - """ - return _invoke_function("ntile", int(_enum_to_value(n))) - - -# ---------------------- Date/Timestamp functions ------------------------------ - - -@_try_remote_functions -def curdate() -> Column: - """ - Returns the current date at the start of query evaluation as a :class:`DateType` column. - All calls of current_date within the same query return the same value. - - .. versionadded:: 3.5.0 - - Returns - ------- - :class:`~pyspark.sql.Column` - current date. - - See Also - -------- - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.current_timestamp` - :meth:`pyspark.sql.functions.localtimestamp` - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.curdate()).show() # doctest: +SKIP - +--------------+ - |current_date()| - +--------------+ - | 2022-08-26| - +--------------+ - """ - return _invoke_function("curdate") - - -@_try_remote_functions -def current_date() -> Column: - """ - Returns the current date at the start of query evaluation as a :class:`DateType` column. - All calls of current_date within the same query return the same value. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Returns - ------- - :class:`~pyspark.sql.Column` - current date. - - See Also - -------- - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.curdate` - :meth:`pyspark.sql.functions.current_timestamp` - :meth:`pyspark.sql.functions.localtimestamp` - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.current_date()).show() # doctest: +SKIP - +--------------+ - |current_date()| - +--------------+ - | 2022-08-26| - +--------------+ - """ - return _invoke_function("current_date") - - -@_try_remote_functions -def current_timezone() -> Column: - """ - Returns the current session local timezone. - - .. versionadded:: 3.5.0 - - Returns - ------- - :class:`~pyspark.sql.Column` - current session local timezone. + current session local timezone. See Also -------- @@ -12790,329 +12201,6 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non return _invoke_function_over_columns("try_to_timestamp", col) -@_try_remote_functions -def xpath(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a string array of values within the nodes of xml that match the XPath expression. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [('b1b2b3c1c2',)], ['x']) - >>> df.select(sf.xpath(df.x, sf.lit('a/b/text()'))).show() - +--------------------+ - |xpath(x, a/b/text())| - +--------------------+ - | [b1, b2, b3]| - +--------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath", xml, path) - - -@_try_remote_functions -def xpath_boolean(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns true if the XPath expression evaluates to true, or if a matching node is found. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('1',)], ['x']) - >>> df.select(sf.xpath_boolean(df.x, sf.lit('a/b'))).show() - +---------------------+ - |xpath_boolean(x, a/b)| - +---------------------+ - | true| - +---------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_boolean", xml, path) - - -@_try_remote_functions -def xpath_double(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a double value, the value zero if no match is found, - or NaN if a match is found but the value is non-numeric. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_double(df.x, sf.lit('sum(a/b)'))).show() - +-------------------------+ - |xpath_double(x, sum(a/b))| - +-------------------------+ - | 3.0| - +-------------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_double", xml, path) - - -@_try_remote_functions -def xpath_number(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a double value, the value zero if no match is found, - or NaN if a match is found but the value is non-numeric. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [('12',)], ['x'] - ... ).select(sf.xpath_number('x', sf.lit('sum(a/b)'))).show() - +-------------------------+ - |xpath_number(x, sum(a/b))| - +-------------------------+ - | 3.0| - +-------------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_number", xml, path) - - -@_try_remote_functions -def xpath_float(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a float value, the value zero if no match is found, - or NaN if a match is found but the value is non-numeric. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_float(df.x, sf.lit('sum(a/b)'))).show() - +------------------------+ - |xpath_float(x, sum(a/b))| - +------------------------+ - | 3.0| - +------------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_float", xml, path) - - -@_try_remote_functions -def xpath_int(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns an integer value, or the value zero if no match is found, - or a match is found but the value is non-numeric. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_int(df.x, sf.lit('sum(a/b)'))).show() - +----------------------+ - |xpath_int(x, sum(a/b))| - +----------------------+ - | 3| - +----------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_int", xml, path) - - -@_try_remote_functions -def xpath_long(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a long integer value, or the value zero if no match is found, - or a match is found but the value is non-numeric. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_long(df.x, sf.lit('sum(a/b)'))).show() - +-----------------------+ - |xpath_long(x, sum(a/b))| - +-----------------------+ - | 3| - +-----------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_long", xml, path) - - -@_try_remote_functions -def xpath_short(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a short integer value, or the value zero if no match is found, - or a match is found but the value is non-numeric. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_short(df.x, sf.lit('sum(a/b)'))).show() - +------------------------+ - |xpath_short(x, sum(a/b))| - +------------------------+ - | 3| - +------------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_short", xml, path) - - -@_try_remote_functions -def xpath_string(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns the text contents of the first xml node that matches the XPath expression. - - .. versionadded:: 3.5.0 - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('bcc',)], ['x']) - >>> df.select(sf.xpath_string(df.x, sf.lit('a/c'))).show() - +--------------------+ - |xpath_string(x, a/c)| - +--------------------+ - | cc| - +--------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - """ - return _invoke_function_over_columns("xpath_string", xml, path) - - @_try_remote_functions def trunc(date: "ColumnOrName", format: str) -> Column: """ @@ -14554,1306 +13642,1838 @@ def to_timestamp_ntz( return _invoke_function_over_columns("to_timestamp_ntz", timestamp) -# ---------------------------- misc functions ---------------------------------- - - @_try_remote_functions -def current_catalog() -> Column: - """Returns the current catalog. +def convert_timezone( + sourceTz: Optional[Column], targetTz: Column, sourceTs: "ColumnOrName" +) -> Column: + """ + Converts the timestamp without time zone `sourceTs` + from the `sourceTz` time zone to `targetTz`. .. versionadded:: 3.5.0 + Parameters + ---------- + sourceTz : :class:`~pyspark.sql.Column`, optional + The time zone for the input timestamp. If it is missed, + the current session time zone is used as the source time zone. + A column that evaluates to a string. + targetTz : :class:`~pyspark.sql.Column` + The time zone to which the input timestamp should be converted. + A column that evaluates to a string. + sourceTs : :class:`~pyspark.sql.Column` or column name + A timestamp without time zone. + A column that evaluates to a timestamp. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains a timestamp for converted time zone. + Returns a column that evaluates to a timestamp. + See Also -------- - :meth:`pyspark.sql.functions.current_database` - :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.current_timezone` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_catalog()).show() - +-----------------+ - |current_catalog()| - +-----------------+ - | spark_catalog| - +-----------------+ - """ - return _invoke_function("current_catalog") - + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") -@_try_remote_functions -def current_path() -> Column: - """Returns the current SQL path as a comma-separated list of qualified schema names. + Example 1: Converts the timestamp without time zone `sourceTs`. - .. versionadded:: 4.2.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('2015-04-08 00:00:00',)], ['ts']) + >>> df.select( + ... '*', + ... sf.convert_timezone(None, sf.lit('Asia/Hong_Kong'), 'ts') + ... ).show() # doctest: +SKIP + +-------------------+--------------------------------------------------------+ + | ts|convert_timezone(current_timezone(), Asia/Hong_Kong, ts)| + +-------------------+--------------------------------------------------------+ + |2015-04-08 00:00:00| 2015-04-08 15:00:00| + +-------------------+--------------------------------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_database` - :meth:`pyspark.sql.functions.current_schema` + Example 2: Converts the timestamp with time zone `sourceTs`. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_path()).show() # doctest: +SKIP - +----------------------------------------------------+ - | current_path()| - +----------------------------------------------------+ - |system.builtin,system.session,spark_catalog.default | - +----------------------------------------------------+ + >>> df = spark.createDataFrame([('2015-04-08 15:00:00',)], ['ts']) + >>> df.select( + ... '*', + ... sf.convert_timezone(sf.lit('Asia/Hong_Kong'), sf.lit('America/Los_Angeles'), df.ts) + ... ).show() + +-------------------+---------------------------------------------------------+ + | ts|convert_timezone(Asia/Hong_Kong, America/Los_Angeles, ts)| + +-------------------+---------------------------------------------------------+ + |2015-04-08 15:00:00| 2015-04-08 00:00:00| + +-------------------+---------------------------------------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - return _invoke_function("current_path") + if sourceTz is None: + return _invoke_function_over_columns("convert_timezone", targetTz, sourceTs) + else: + return _invoke_function_over_columns("convert_timezone", sourceTz, targetTz, sourceTs) @_try_remote_functions -def current_database() -> Column: - """Returns the current database. +def make_dt_interval( + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, +) -> Column: + """ + Make DayTimeIntervalType duration from days, hours, mins and secs. .. versionadded:: 3.5.0 + Parameters + ---------- + days : :class:`~pyspark.sql.Column` or column name, optional + The number of days, positive or negative. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The number of hours, positive or negative. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The number of minutes, positive or negative. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains a DayTimeIntervalType duration. + Returns a column that evaluates to an interval. + See Also -------- - :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.make_ym_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_database()).show() - +----------------+ - |current_schema()| - +----------------+ - | default| - +----------------+ - """ - return _invoke_function("current_database") + Example 1: Make DayTimeIntervalType duration from days, hours, mins and secs. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval(df.day, df.hour, df.min, df.sec)).show(truncate=False) + +---+----+---+--------+------------------------------------------+ + |day|hour|min|sec |make_dt_interval(day, hour, min, sec) | + +---+----+---+--------+------------------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 12:30:01.001001' DAY TO SECOND| + +---+----+---+--------+------------------------------------------+ -@_try_remote_functions -def current_schema() -> Column: - """Returns the current database. + Example 2: Make DayTimeIntervalType duration from days, hours and mins. - .. versionadded:: 3.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval(df.day, 'hour', df.min)).show(truncate=False) + +---+----+---+--------+-----------------------------------+ + |day|hour|min|sec |make_dt_interval(day, hour, min, 0)| + +---+----+---+--------+-----------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 12:30:00' DAY TO SECOND| + +---+----+---+--------+-----------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_database` + Example 3: Make DayTimeIntervalType duration from days and hours. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_schema()).show() - +----------------+ - |current_schema()| - +----------------+ - | default| - +----------------+ - """ - return _invoke_function("current_schema") - + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval(df.day, df.hour)).show(truncate=False) + +---+----+---+--------+-----------------------------------+ + |day|hour|min|sec |make_dt_interval(day, hour, 0, 0) | + +---+----+---+--------+-----------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 12:00:00' DAY TO SECOND| + +---+----+---+--------+-----------------------------------+ -@_try_remote_functions -def current_user() -> Column: - """Returns the current database. + Example 4: Make DayTimeIntervalType duration from days. - .. versionadded:: 3.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval('day')).show(truncate=False) + +---+----+---+--------+-----------------------------------+ + |day|hour|min|sec |make_dt_interval(day, 0, 0, 0) | + +---+----+---+--------+-----------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 00:00:00' DAY TO SECOND| + +---+----+---+--------+-----------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.session_user` + Example 5: Make empty interval. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_user()).show() # doctest: +SKIP - +--------------+ - |current_user()| - +--------------+ - | ruifeng.zheng| - +--------------+ + >>> spark.range(1).select(sf.make_dt_interval()).show(truncate=False) + +-----------------------------------+ + |make_dt_interval(0, 0, 0, 0) | + +-----------------------------------+ + |INTERVAL '0 00:00:00' DAY TO SECOND| + +-----------------------------------+ """ - return _invoke_function("current_user") + _days = lit(0) if days is None else days + _hours = lit(0) if hours is None else hours + _mins = lit(0) if mins is None else mins + _secs = lit(decimal.Decimal(0)) if secs is None else secs + return _invoke_function_over_columns("make_dt_interval", _days, _hours, _mins, _secs) @_try_remote_functions -def user() -> Column: - """Returns the current database. +def try_make_interval( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + weeks: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, +) -> Column: + """ + This is a special version of `make_interval` that performs the same operation, but returns a + NULL value instead of raising an error if interval cannot be created. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 + + Parameters + ---------- + years : :class:`~pyspark.sql.Column` or column name, optional + The number of years, positive or negative. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The number of months, positive or negative. + A column that evaluates to an integer. + weeks : :class:`~pyspark.sql.Column` or column name, optional + The number of weeks, positive or negative. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The number of days, positive or negative. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The number of hours, positive or negative. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The number of minutes, positive or negative. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains an interval. + Returns a column that evaluates to an interval. See Also -------- - :meth:`pyspark.sql.functions.current_user` - :meth:`pyspark.sql.functions.session_user` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.make_dt_interval` + :meth:`pyspark.sql.functions.make_ym_interval` Examples -------- + Example 1: Try make interval from years, months, weeks, days, hours, mins and secs. + >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.user()).show() # doctest: +SKIP - +--------------+ - | user()| - +--------------+ - | ruifeng.zheng| - +--------------+ - """ - return _invoke_function("user") + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_interval(df.year, df.month, 'week', df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +---------------------------------------------------------------+ + |try_make_interval(year, month, week, day, hour, min, sec) | + +---------------------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| + +---------------------------------------------------------------+ + Example 2: Try make interval from years, months, weeks, days, hours and mins. -@_try_remote_functions -def session_user() -> Column: - """Returns the user name of current execution context. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) + ... ).show(truncate=False) + +-------------------------------------------------------+ + |try_make_interval(year, month, week, day, hour, min, 0)| + +-------------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes | + +-------------------------------------------------------+ - .. versionadded:: 4.0.0 + Example 3: Try make interval from years, months, weeks, days and hours. - See Also - -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.current_user` + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour) + ... ).show(truncate=False) + +-----------------------------------------------------+ + |try_make_interval(year, month, week, day, hour, 0, 0)| + +-----------------------------------------------------+ + |100 years 11 months 8 days 12 hours | + +-----------------------------------------------------+ + + Example 4: Try make interval from years, months, weeks and days. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.session_user()).show() # doctest: +SKIP - +--------------+ - |session_user()| - +--------------+ - | ruifeng.zheng| - +--------------+ - """ - return _invoke_function("session_user") + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year, 'month', df.week, df.day)).show(truncate=False) + +--------------------------------------------------+ + |try_make_interval(year, month, week, day, 0, 0, 0)| + +--------------------------------------------------+ + |100 years 11 months 8 days | + +--------------------------------------------------+ + Example 5: Try make interval from years, months and weeks. -@_try_remote_functions -def uuid(seed: Optional[Union[Column, int]] = None) -> Column: - """Returns an universally unique identifier (UUID) string. - The value is returned as a canonical UUID 36-character string. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year, 'month', df.week)).show(truncate=False) + +------------------------------------------------+ + |try_make_interval(year, month, week, 0, 0, 0, 0)| + +------------------------------------------------+ + |100 years 11 months 7 days | + +------------------------------------------------+ - .. versionadded:: 4.1.0 + Example 6: Try make interval from years and months. - Parameters - ---------- - seed : :class:`~pyspark.sql.Column` or int - Optional random number seed to use. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year, 'month')).show(truncate=False) + +---------------------------------------------+ + |try_make_interval(year, month, 0, 0, 0, 0, 0)| + +---------------------------------------------+ + |100 years 11 months | + +---------------------------------------------+ - Examples - -------- - Example 1: Generate UUIDs with random seed + Example 7: Try make interval from years. - >>> from pyspark.sql import functions as sf - >>> spark.range(5).select(sf.uuid()).show(truncate=False) # doctest: +SKIP - +------------------------------------+ - |uuid() | - +------------------------------------+ - |627ae05e-b319-42b5-b4e4-71c8c9754dd1| - |f781cce5-a2e2-464d-bc8b-426ff448e404| - |15e2e66e-8416-4ea2-af3c-409363408189| - |fb1d6178-7676-4791-baa9-f2ddcc494515| - |d48665e8-2657-4c6b-b7c8-8ae0cd646e41| - +------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year)).show(truncate=False) + +-----------------------------------------+ + |try_make_interval(year, 0, 0, 0, 0, 0, 0)| + +-----------------------------------------+ + |100 years | + +-----------------------------------------+ - Example 2: Generate UUIDs with a specified seed + Example 8: Try make empty interval. - >>> from pyspark.sql import functions as sf - >>> spark.range(0, 5, 1, 1).select(sf.uuid(seed=123)).show(truncate=False) - +------------------------------------+ - |uuid() | - +------------------------------------+ - |4c99192d-23d6-4d88-b814-a634398120f0| - |af506873-3c53-41e3-8354-a24856b8de8a| - |7b4b370e-e867-47e2-93c0-f6990463a12d| - |1c4d1733-ff1a-4a6c-b144-0b0345adf0d0| - |7478f235-f8bc-4112-8e59-a28f50e46890| - +------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.try_make_interval()).show(truncate=False) + +--------------------------------------+ + |try_make_interval(0, 0, 0, 0, 0, 0, 0)| + +--------------------------------------+ + |0 seconds | + +--------------------------------------+ - if seed is None: - return _invoke_function("uuid") - else: - return _invoke_function("uuid", _to_java_column(lit(seed))) + Example 9: Try make interval from years with overflow. + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.try_make_interval(sf.lit(2147483647))).show(truncate=False) + +-----------------------------------------------+ + |try_make_interval(2147483647, 0, 0, 0, 0, 0, 0)| + +-----------------------------------------------+ + |NULL | + +-----------------------------------------------+ + """ + _years = lit(0) if years is None else years + _months = lit(0) if months is None else months + _weeks = lit(0) if weeks is None else weeks + _days = lit(0) if days is None else days + _hours = lit(0) if hours is None else hours + _mins = lit(0) if mins is None else mins + _secs = lit(decimal.Decimal(0)) if secs is None else secs + return _invoke_function_over_columns( + "try_make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs + ) @_try_remote_functions -def crc32(col: "ColumnOrName") -> Column: +def make_interval( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + weeks: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, +) -> Column: """ - Calculates the cyclic redundancy check value (CRC32) of a binary column and - returns the value as a bigint. + Make interval from years, months, weeks, days, hours, mins and secs. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. + years : :class:`~pyspark.sql.Column` or column name, optional + The number of years, positive or negative. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The number of months, positive or negative. + A column that evaluates to an integer. + weeks : :class:`~pyspark.sql.Column` or column name, optional + The number of weeks, positive or negative. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The number of days, positive or negative. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The number of hours, positive or negative. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The number of minutes, positive or negative. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. - Returns a column that evaluates to a long. + A new column that contains an interval. + Returns a column that evaluates to an interval. - .. versionadded:: 1.5.0 + See Also + -------- + :meth:`pyspark.sql.functions.make_dt_interval` + :meth:`pyspark.sql.functions.make_ym_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- + Example 1: Make interval from years, months, weeks, days, hours, mins and secs. + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC',)], ['a']) - >>> df.select('*', sf.crc32('a')).show(truncate=False) - +---+----------+ - |a |crc32(a) | - +---+----------+ - |ABC|2743272264| - +---+----------+ - """ - return _invoke_function_over_columns("crc32", col) + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +---------------------------------------------------------------+ + |make_interval(year, month, week, day, hour, min, sec) | + +---------------------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| + +---------------------------------------------------------------+ + Example 2: Make interval from years, months, weeks, days, hours and mins. -@_try_remote_functions -def md5(col: "ColumnOrName") -> Column: - """Calculates the MD5 digest and returns the value as a 32 character hex string. - - .. versionadded:: 1.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) + ... ).show(truncate=False) + +---------------------------------------------------+ + |make_interval(year, month, week, day, hour, min, 0)| + +---------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes | + +---------------------------------------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Make interval from years, months, weeks, days and hours. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour) + ... ).show(truncate=False) + +-------------------------------------------------+ + |make_interval(year, month, week, day, hour, 0, 0)| + +-------------------------------------------------+ + |100 years 11 months 8 days 12 hours | + +-------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. - Returns a column that evaluates to a string. + Example 4: Make interval from years, months, weeks and days. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC',)], ['a']) - >>> df.select('*', sf.md5('a')).show(truncate=False) - +---+--------------------------------+ - |a |md5(a) | - +---+--------------------------------+ - |ABC|902fbdd2b1df0c4f70b4a5d23525e932| - +---+--------------------------------+ - """ - return _invoke_function_over_columns("md5", col) + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year, df.month, 'week', df.day)).show(truncate=False) + +----------------------------------------------+ + |make_interval(year, month, week, day, 0, 0, 0)| + +----------------------------------------------+ + |100 years 11 months 8 days | + +----------------------------------------------+ + Example 5: Make interval from years, months and weeks. -@_try_remote_functions -def xxh3_64(col: "ColumnOrName") -> Column: - """Returns a 64-bit hash value of the argument using the XXH3 algorithm. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year, df.month, 'week')).show(truncate=False) + +--------------------------------------------+ + |make_interval(year, month, week, 0, 0, 0, 0)| + +--------------------------------------------+ + |100 years 11 months 7 days | + +--------------------------------------------+ - Unlike :func:`xxhash64`, which hashes one or more columns structurally, this hashes the raw - bytes of a single value with seed 0, so its result is byte compatible with the reference XXH3. + Example 6: Make interval from years and months. - .. versionadded:: 4.4.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year, df.month)).show(truncate=False) + +-----------------------------------------+ + |make_interval(year, month, 0, 0, 0, 0, 0)| + +-----------------------------------------+ + |100 years 11 months | + +-----------------------------------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column to hash, which must have string or binary type. + Example 7: Make interval from years. - Returns - ------- - :class:`~pyspark.sql.Column` - Returns a column that evaluates to a long. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year)).show(truncate=False) + +-------------------------------------+ + |make_interval(year, 0, 0, 0, 0, 0, 0)| + +-------------------------------------+ + |100 years | + +-------------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.xxh3_128` - :meth:`pyspark.sql.functions.xxhash64` + Example 8: Make empty interval. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark',)], ['a']) - >>> df.select(sf.xxh3_64('a').alias('h')).collect() - [Row(h=80997306238743657)] + >>> spark.range(1).select(sf.make_interval()).show(truncate=False) + +----------------------------------+ + |make_interval(0, 0, 0, 0, 0, 0, 0)| + +----------------------------------+ + |0 seconds | + +----------------------------------+ """ - return _invoke_function_over_columns("xxh3_64", col) + _years = lit(0) if years is None else years + _months = lit(0) if months is None else months + _weeks = lit(0) if weeks is None else weeks + _days = lit(0) if days is None else days + _hours = lit(0) if hours is None else hours + _mins = lit(0) if mins is None else mins + _secs = lit(decimal.Decimal(0)) if secs is None else secs + return _invoke_function_over_columns( + "make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs + ) @_try_remote_functions -def xxh3_128(col: "ColumnOrName") -> Column: - """Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. +def make_time(hour: "ColumnOrName", minute: "ColumnOrName", second: "ColumnOrName") -> Column: + """ + Create time from hour, minute and second fields. For invalid inputs it will throw an error. - .. versionadded:: 4.4.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column to hash, which must have string or binary type. + hour : :class:`~pyspark.sql.Column` or column name + The hour to represent, from 0 to 23. + A column that evaluates to an integer. + minute : :class:`~pyspark.sql.Column` or column name + The minute to represent, from 0 to 59. + A column that evaluates to an integer. + second : :class:`~pyspark.sql.Column` or column name + The second to represent, from 0 to 59.999999. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.xxh3_64` - :meth:`pyspark.sql.functions.md5` + A column representing the created time. + Returns a column that evaluates to a time. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark',)], ['a']) - >>> df.select(sf.xxh3_128('a').alias('h')).collect() - [Row(h='7d57dd84c60c86ca1f4e82ab91a12b5e')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(6, 30, 45.887)], ["hour", "minute", "second"]) + >>> df.select(sf.make_time("hour", "minute", "second").alias("time")).show() + +------------+ + | time| + +------------+ + |06:30:45.887| + +------------+ """ - return _invoke_function_over_columns("xxh3_128", col) + return _invoke_function_over_columns("make_time", hour, minute, second) @_try_remote_functions -def sha1(col: "ColumnOrName") -> Column: - """Returns the hex string result of SHA-1. - - .. versionadded:: 1.5.0 +def time_from_seconds(col: "ColumnOrName") -> Column: + """ + Creates a TIME value from seconds since midnight (supports fractional seconds). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.sha` - :meth:`pyspark.sql.functions.sha2` + Seconds since midnight (0 to 86399.999999). + A column that evaluates to a numeric. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC',)], ['a']) - >>> df.select('*', sf.sha1('a')).show(truncate=False) - +---+----------------------------------------+ - |a |sha1(a) | - +---+----------------------------------------+ - |ABC|3c01bdbb26f358bab27f267924aa2c9a03fcfdb8| - +---+----------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(52200.5,)], ['seconds']) + >>> df.select(sf.time_from_seconds('seconds')).show() + +--------------------------+ + |time_from_seconds(seconds)| + +--------------------------+ + | 14:30:00.5| + +--------------------------+ """ - return _invoke_function_over_columns("sha1", col) + return _invoke_function_over_columns("time_from_seconds", col) @_try_remote_functions -def sha2(col: "ColumnOrName", numBits: int) -> Column: - """Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, - and SHA-512). The numBits indicates the desired bit length of the result, which must have a - value of 224, 256, 384, 512, or 0 (which is equivalent to 256). - - .. versionadded:: 1.5.0 +def time_from_millis(col: "ColumnOrName") -> Column: + """ + Creates a TIME value from milliseconds since midnight. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. - numBits : int - the desired bit length of the result, which must have a - value of 224, 256, 384, 512, or 0 (which is equivalent to 256). - A column that evaluates to an integer. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.sha` - :meth:`pyspark.sql.functions.sha1` + Milliseconds since midnight (0 to 86399999). + A column that evaluates to an integral. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([['Alice'], ['Bob']], ['name']) - >>> df.select('*', sf.sha2('name', 256)).show(truncate=False) - +-----+----------------------------------------------------------------+ - |name |sha2(name, 256) | - +-----+----------------------------------------------------------------+ - |Alice|3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043| - |Bob |cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961| - +-----+----------------------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(52200500,)], ['millis']) + >>> df.select(sf.time_from_millis('millis')).show() + +------------------------+ + |time_from_millis(millis)| + +------------------------+ + | 14:30:00.5| + +------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if numBits not in [0, 224, 256, 384, 512]: - raise PySparkValueError( - errorClass="VALUE_NOT_ALLOWED", - messageParameters={ - "arg_name": "numBits", - "allowed_values": "[0, 224, 256, 384, 512]", - }, - ) - return _invoke_function("sha2", _to_java_column(col), numBits) + return _invoke_function_over_columns("time_from_millis", col) @_try_remote_functions -def hash(*cols: "ColumnOrName") -> Column: - """Calculates the hash code of given columns, and returns the result as an int column. - - .. versionadded:: 2.0.0 +def time_from_micros(col: "ColumnOrName") -> Column: + """ + Creates a TIME value from microseconds since midnight. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - one or more columns to compute on. - Each a column of any type. - - Returns - ------- - :class:`~pyspark.sql.Column` - hash value as int column. - Returns a column that evaluates to an integer. - - See Also - -------- - :meth:`pyspark.sql.functions.xxhash64` + col : :class:`~pyspark.sql.Column` or column name + Microseconds since midnight (0 to 86399999999). + A column that evaluates to an integral. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) - >>> df.select('*', sf.hash('c1')).show() - +---+---+----------+ - | c1| c2| hash(c1)| - +---+---+----------+ - |ABC|DEF|-757602832| - +---+---+----------+ - - >>> df.select('*', sf.hash('c1', df.c2)).show() - +---+---+------------+ - | c1| c2|hash(c1, c2)| - +---+---+------------+ - |ABC|DEF| 599895104| - +---+---+------------+ - - >>> df.select('*', sf.hash('*')).show() - +---+---+------------+ - | c1| c2|hash(c1, c2)| - +---+---+------------+ - |ABC|DEF| 599895104| - +---+---+------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(52200500000,)], ['micros']) + >>> df.select(sf.time_from_micros('micros')).show() + +------------------------+ + |time_from_micros(micros)| + +------------------------+ + | 14:30:00.5| + +------------------------+ """ - return _invoke_function_over_seq_of_columns("hash", cols) + return _invoke_function_over_columns("time_from_micros", col) @_try_remote_functions -def xxhash64(*cols: "ColumnOrName") -> Column: - """Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, - and returns the result as a long column. The hash computation uses an initial seed of 42. - - .. versionadded:: 3.0.0 +def time_to_seconds(col: "ColumnOrName") -> Column: + """ + Extracts seconds from TIME value (returns DECIMAL to preserve fractional seconds). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - one or more columns to compute on. - Each a column of any type. - - Returns - ------- - :class:`~pyspark.sql.Column` - hash value as long column. - Returns a column that evaluates to a long. - - See Also - -------- - :meth:`pyspark.sql.functions.hash` - :meth:`pyspark.sql.functions.xxh3_64` + col : :class:`~pyspark.sql.Column` or column name + TIME value to convert. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) - >>> df.select('*', sf.xxhash64('c1')).show() - +---+---+-------------------+ - | c1| c2| xxhash64(c1)| - +---+---+-------------------+ - |ABC|DEF|4105715581806190027| - +---+---+-------------------+ - - >>> df.select('*', sf.xxhash64('c1', df.c2)).show() - +---+---+-------------------+ - | c1| c2| xxhash64(c1, c2)| - +---+---+-------------------+ - |ABC|DEF|3233247871021311208| - +---+---+-------------------+ - - >>> df.select('*', sf.xxhash64('*')).show() - +---+---+-------------------+ - | c1| c2| xxhash64(c1, c2)| - +---+---+-------------------+ - |ABC|DEF|3233247871021311208| - +---+---+-------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") + >>> df.select(sf.time_to_seconds('time')).show() + +---------------------+ + |time_to_seconds(time)| + +---------------------+ + | 52200.500000| + +---------------------+ """ - return _invoke_function_over_seq_of_columns("xxhash64", cols) + return _invoke_function_over_columns("time_to_seconds", col) @_try_remote_functions -def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None) -> Column: +def time_to_millis(col: "ColumnOrName") -> Column: """ - Returns `null` if the input column is `true`; throws an exception - with the provided error message otherwise. - - .. versionadded:: 3.1.0 + Extracts milliseconds from TIME value. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column name or column that represents the input column to test. - A column that evaluates to a boolean. - errMsg : :class:`~pyspark.sql.Column` or literal string, optional - A Python string literal or column containing the error message. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - `null` if the input column is `true` otherwise throws an error with specified message. - Returns a column that always evaluates to NULL. - - See Also - -------- - :meth:`pyspark.sql.functions.raise_error` + TIME value to convert. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(0, 1)], ['a', 'b']) - >>> df.select('*', sf.assert_true(df.a < df.b)).show() - +---+---+--------------------------------------------+ - | a| b|assert_true((a < b), '(a < b)' is not true!)| - +---+---+--------------------------------------------+ - | 0| 1| NULL| - +---+---+--------------------------------------------+ - - >>> df.select('*', sf.assert_true(df.a < df.b, df.a)).show() - +---+---+-----------------------+ - | a| b|assert_true((a < b), a)| - +---+---+-----------------------+ - | 0| 1| NULL| - +---+---+-----------------------+ - - >>> df.select('*', sf.assert_true(df.a < df.b, 'error')).show() - +---+---+---------------------------+ - | a| b|assert_true((a < b), error)| - +---+---+---------------------------+ - | 0| 1| NULL| - +---+---+---------------------------+ - - >>> df.select('*', sf.assert_true(df.a > df.b, 'My error msg')).show() # doctest: +SKIP - ... - java.lang.RuntimeException: My error msg - ... + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") + >>> df.select(sf.time_to_millis('time')).show() + +--------------------+ + |time_to_millis(time)| + +--------------------+ + | 52200500| + +--------------------+ """ - errMsg = _enum_to_value(errMsg) - if errMsg is None: - return _invoke_function_over_columns("assert_true", col) - if not isinstance(errMsg, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "errMsg", - "arg_type": type(errMsg).__name__, - }, - ) - return _invoke_function_over_columns("assert_true", col, lit(errMsg)) + return _invoke_function_over_columns("time_to_millis", col) @_try_remote_functions -def raise_error(errMsg: Union[Column, str]) -> Column: +def time_to_micros(col: "ColumnOrName") -> Column: """ - Throws an exception with the provided error message. - - .. versionadded:: 3.1.0 + Extracts microseconds from TIME value. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- - errMsg : :class:`~pyspark.sql.Column` or literal string - A Python string literal or column containing the error message. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - throws an error with specified message. - Returns a column that always evaluates to NULL. - - See Also - -------- - :meth:`pyspark.sql.functions.assert_true` + col : :class:`~pyspark.sql.Column` or column name + TIME value to convert. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.raise_error("My error message")).show() # doctest: +SKIP - ... - java.lang.RuntimeException: My error message - ... + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") + >>> df.select(sf.time_to_micros('time')).show() + +--------------------+ + |time_to_micros(time)| + +--------------------+ + | 52200500000| + +--------------------+ """ - errMsg = _enum_to_value(errMsg) - if not isinstance(errMsg, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "errMsg", - "arg_type": type(errMsg).__name__, - }, - ) - return _invoke_function_over_columns("raise_error", lit(errMsg)) - - -# ---------------------- String/Binary functions ------------------------------ + return _invoke_function_over_columns("time_to_micros", col) -@_try_remote_functions -def upper(col: "ColumnOrName") -> Column: - """ - Converts a string expression to upper case. +@overload +def make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... - .. versionadded:: 1.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@overload +def make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: "ColumnOrName", +) -> Column: ... - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - Returns - ------- - :class:`~pyspark.sql.Column` - upper case values. - Returns a column that evaluates to a string. +@overload +def make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... - See Also - -------- - :meth:`pyspark.sql.functions.lower` - :meth:`pyspark.sql.functions.ucase` - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.upper("value")).show() - +----------+------------+ - | value|upper(value)| - +----------+------------+ - | Spark| SPARK| - | PySpark| PYSPARK| - |Pandas API| PANDAS API| - +----------+------------+ - """ - return _invoke_function_over_columns("upper", col) +@overload +def make_timestamp( + *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" +) -> Column: ... @_try_remote_functions -def lower(col: "ColumnOrName") -> Column: +def make_timestamp( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + timezone: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Converts a string expression to lower case. + Create timestamp from years, months, days, hours, mins, secs, and (optional) timezone fields. + Alternatively, create timestamp from date, time, and (optional) timezone fields. + The result data type is consistent with the value of configuration `spark.sql.timestampType`. + If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL + on invalid inputs. Otherwise, it will throw an error instead. - .. versionadded:: 1.5.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or column name, optional + The time zone identifier. For example, CET, UTC, and etc. A column that evaluates to a string. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - lower case values. - Returns a column that evaluates to a string. + A new column that contains a timestamp. + Returns a column that evaluates to a timestamp. See Also -------- - :meth:`pyspark.sql.functions.upper` - :meth:`pyspark.sql.functions.lcase` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.lower("value")).show() - +----------+------------+ - | value|lower(value)| - +----------+------------+ - | Spark| spark| - | PySpark| pyspark| - |Pandas API| pandas api| - +----------+------------+ - """ - return _invoke_function_over_columns("lower", col) + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. -@_try_remote_functions -def ascii(col: "ColumnOrName") -> Column: - """ - Computes the numeric value of the first character of the string column. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') + ... ).show(truncate=False) + +----------------------------------------------------+ + |make_timestamp(year, month, day, hour, min, sec, tz)| + +----------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +----------------------------------------------------+ - .. versionadded:: 1.5.0 + Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +------------------------------------------------+ + |make_timestamp(year, month, day, hour, min, sec)| + +------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +------------------------------------------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + Example 3: Make timestamp from date, time, and timezone. - Returns - ------- - :class:`~pyspark.sql.Column` - numeric value. - Returns a column that evaluates to an integer. + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time"), + ... sf.lit("CET").alias("tz") + ... ) + >>> df.select( + ... sf.make_timestamp(date=df.date, time=df.time, timezone=df.tz) + ... ).show(truncate=False) + +------------------------------+ + |make_timestamp(date, time, tz)| + +------------------------------+ + |2014-12-27 21:30:45.887 | + +------------------------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.ascii("value")).show() - +----------+------------+ - | value|ascii(value)| - +----------+------------+ - | Spark| 83| - | PySpark| 80| - |Pandas API| 80| - +----------+------------+ - """ - return _invoke_function_over_columns("ascii", col) + Example 4: Make timestamp from date and time (without timezone). + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time") + ... ) + >>> df.select(sf.make_timestamp(date=df.date, time=df.time)).show(truncate=False) + +--------------------------+ + |make_timestamp(date, time)| + +--------------------------+ + |2014-12-28 06:30:45.887 | + +--------------------------+ -@_try_remote_functions -def base64(col: "ColumnOrName") -> Column: + >>> spark.conf.unset("spark.sql.session.timeZone") """ - Computes the BASE64 encoding of a binary column and returns it as a string column. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a binary. - - Returns - ------- - :class:`~pyspark.sql.Column` - BASE64 encoding of string value. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.unbase64` - :meth:`pyspark.sql.functions.to_base32` + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) + else: + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "make_timestamp", + _ensure_column_or_name(date), + _ensure_column_or_name(time), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.base64("value")).show() - +----------+----------------+ - | value| base64(value)| - +----------+----------------+ - | Spark| U3Bhcms=| - | PySpark| UHlTcGFyaw==| - |Pandas API|UGFuZGFzIEFQSQ==| - +----------+----------------+ - """ - return _invoke_function_over_columns("base64", col) +@overload +def try_make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... -@_try_remote_functions -def to_base32(col: "ColumnOrName") -> Column: - """ - Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a - string column. - .. versionadded:: 4.3.0 +@overload +def try_make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: "ColumnOrName", +) -> Column: ... - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a binary. - Returns - ------- - :class:`~pyspark.sql.Column` - BASE32 encoding of the binary value. - Returns a column that evaluates to a string. +@overload +def try_make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... - See Also - -------- - :meth:`pyspark.sql.functions.from_base32` - :meth:`pyspark.sql.functions.base64` - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(b"foobar",)], ["value"]) - >>> df.select(sf.to_base32("value").alias("r")).collect() - [Row(r='MZXW6YTBOI======')] - """ - return _invoke_function_over_columns("to_base32", col) +@overload +def try_make_timestamp( + *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" +) -> Column: ... @_try_remote_functions -def unbase64(col: "ColumnOrName") -> Column: +def try_make_timestamp( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + timezone: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Decodes a BASE64 encoded string column and returns it as a binary column. + Try to create timestamp from years, months, days, hours, mins, secs and (optional) timezone + fields. Alternatively, try to create timestamp from date, time, and (optional) timezone fields. + The result data type is consistent with the value of configuration `spark.sql.timestampType`. + The function returns NULL on invalid inputs. - .. versionadded:: 1.5.0 + .. versionadded:: 4.0.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or column name, optional + The time zone identifier. For example, CET, UTC, and etc. A column that evaluates to a string. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - decoded binary value. - Returns a column that evaluates to a binary. + A new column that contains a timestamp or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- - :meth:`pyspark.sql.functions.base64` - :meth:`pyspark.sql.functions.from_base32` + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["U3Bhcms=", "UHlTcGFyaw==", "UGFuZGFzIEFQSQ=="], "STRING") - >>> df.select("*", sf.unbase64("value")).show(truncate=False) - +----------------+-------------------------------+ - |value |unbase64(value) | - +----------------+-------------------------------+ - |U3Bhcms= |[53 70 61 72 6B] | - |UHlTcGFyaw== |[50 79 53 70 61 72 6B] | - |UGFuZGFzIEFQSQ==|[50 61 6E 64 61 73 20 41 50 49]| - +----------------+-------------------------------+ - """ - return _invoke_function_over_columns("unbase64", col) - - -@_try_remote_functions -def from_base32(col: "ColumnOrName") -> Column: - """ - Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary - column. + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - .. versionadded:: 4.3.0 + Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') + ... ).show(truncate=False) + +----------------------------------------------------+ + |try_make_timestamp(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +----------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - decoded binary value. - Returns a column that evaluates to a binary. + Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). - See Also - -------- - :meth:`pyspark.sql.functions.to_base32` - :meth:`pyspark.sql.functions.unbase64` + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |try_make_timestamp(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +----------------------------------------------------+ + + Example 3: Make timestamp with invalid input. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("MZXW6YTBOI======",)], ["value"]) - >>> df.select(sf.from_base32("value").alias("r")).collect() - [Row(r=b'foobar')] + >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |try_make_timestamp(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |NULL | + +----------------------------------------------------+ + + Example 4: Make timestamp from date, time, and timezone. + + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time"), + ... sf.lit("CET").alias("tz") + ... ) + >>> df.select( + ... sf.try_make_timestamp(date=df.date, time=df.time, timezone=df.tz) + ... ).show(truncate=False) + +----------------------------------+ + |try_make_timestamp(date, time, tz)| + +----------------------------------+ + |2014-12-27 21:30:45.887 | + +----------------------------------+ + + Example 5: Make timestamp from date and time (without timezone). + + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time") + ... ) + >>> df.select(sf.try_make_timestamp(date=df.date, time=df.time)).show(truncate=False) + +------------------------------+ + |try_make_timestamp(date, time)| + +------------------------------+ + |2014-12-28 06:30:45.887 | + +------------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - return _invoke_function_over_columns("from_base32", col) + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "try_make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "try_make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) + else: + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "try_make_timestamp", + _ensure_column_or_name(date), + _ensure_column_or_name(time), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "try_make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) @_try_remote_functions -def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: +def make_timestamp_ltz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: Optional["ColumnOrName"] = None, +) -> Column: """ - Trim the spaces from left end for the specified string value. - - .. versionadded:: 1.5.0 + Create the current timestamp with local time zone from years, months, days, hours, mins, + secs and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, + the function returns NULL on invalid inputs. Otherwise, it will throw an error instead. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space. + years : :class:`~pyspark.sql.Column` or str + The year to represent, from 1 to 9999. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or str + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or str + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or str + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or str + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or str + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13 , or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or str, optional + The time zone identifier. For example, CET, UTC and etc. A column that evaluates to a string. - .. versionadded:: 4.0.0 - Returns ------- :class:`~pyspark.sql.Column` - left trimmed values. - Returns a column that evaluates to a string. + A new column that contains a current timestamp. + Returns a column that evaluates to a timestamp. See Also -------- - :meth:`pyspark.sql.functions.trim` - :meth:`pyspark.sql.functions.rtrim` + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - Example 1: Trim the spaces + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") - >>> df.select("*", sf.ltrim("value")).show() - +--------+------------+ - | value|ltrim(value)| - +--------+------------+ - | Spark| Spark| - | Spark | Spark | - | Spark| Spark| - +--------+------------+ + Example 1: Make the current timestamp from years, months, days, hours, mins and secs. - Example 2: Trim specified characters + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec, 'tz') + ... ).show(truncate=False) + +--------------------------------------------------------+ + |make_timestamp_ltz(year, month, day, hour, min, sec, tz)| + +--------------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +--------------------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") - >>> df.select("*", sf.ltrim("value", sf.lit("*"))).show() - +--------+--------------------------+ - | value|TRIM(LEADING * FROM value)| - +--------+--------------------------+ - |***Spark| Spark| - | Spark**| Spark**| - | *Spark| Spark| - +--------+--------------------------+ + Example 2: Make the current timestamp without timezone. - Example 3: Trim a column containing different characters + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |make_timestamp_ltz(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +----------------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) - >>> df.select("*", sf.ltrim("value", "t")).show() - +--------+---+--------------------------+ - | value| t|TRIM(LEADING t FROM value)| - +--------+---+--------------------------+ - |**Spark*| *| Spark*| - |==Spark=| =| Spark=| - +--------+---+--------------------------+ + >>> spark.conf.unset("spark.sql.session.timeZone") """ - if trim is not None: - return _invoke_function_over_columns("ltrim", col, trim) + if timezone is not None: + return _invoke_function_over_columns( + "make_timestamp_ltz", years, months, days, hours, mins, secs, timezone + ) else: - return _invoke_function_over_columns("ltrim", col) + return _invoke_function_over_columns( + "make_timestamp_ltz", years, months, days, hours, mins, secs + ) @_try_remote_functions -def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: +def try_make_timestamp_ltz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: Optional["ColumnOrName"] = None, +) -> Column: """ - Trim the spaces from right end for the specified string value. - - .. versionadded:: 1.5.0 + Try to create the current timestamp with local time zone from years, months, days, hours, mins, + secs and timezone fields. + The function returns NULL on invalid inputs. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space. + years : :class:`~pyspark.sql.Column` or column name + The year to represent, from 1 to 9999. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13 , or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or column name, optional + The time zone identifier. For example, CET, UTC and etc. A column that evaluates to a string. - .. versionadded:: 4.0.0 - Returns ------- :class:`~pyspark.sql.Column` - right trimmed values. - Returns a column that evaluates to a string. + A new column that contains a current timestamp, or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- - :meth:`pyspark.sql.functions.trim` - :meth:`pyspark.sql.functions.ltrim` + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - Example 1: Trim the spaces + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") - >>> df.select("*", sf.rtrim("value")).show() - +--------+------------+ - | value|rtrim(value)| - +--------+------------+ - | Spark| Spark| - | Spark | Spark| - | Spark| Spark| - +--------+------------+ + Example 1: Make the current timestamp from years, months, days, hours, mins and secs. - Example 2: Trim specified characters + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec, 'tz') + ... ).show(truncate=False) + +------------------------------------------------------------+ + |try_make_timestamp_ltz(year, month, day, hour, min, sec, tz)| + +------------------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +------------------------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") - >>> df.select("*", sf.rtrim("value", sf.lit("*"))).show() - +--------+---------------------------+ - | value|TRIM(TRAILING * FROM value)| - +--------+---------------------------+ - |***Spark| ***Spark| - | Spark**| Spark| - | *Spark| *Spark| - +--------+---------------------------+ + Example 2: Make the current timestamp without timezone. - Example 3: Trim a column containing different characters + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ltz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +--------------------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) - >>> df.select("*", sf.rtrim("value", "t")).show() - +--------+---+---------------------------+ - | value| t|TRIM(TRAILING t FROM value)| - +--------+---+---------------------------+ - |**Spark*| *| **Spark| - |==Spark=| =| ==Spark| - +--------+---+---------------------------+ + Example 3: Make the current timestamp with invalid input. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ltz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |NULL | + +--------------------------------------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - if trim is not None: - return _invoke_function_over_columns("rtrim", col, trim) + if timezone is not None: + return _invoke_function_over_columns( + "try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone + ) else: - return _invoke_function_over_columns("rtrim", col) + return _invoke_function_over_columns( + "try_make_timestamp_ltz", years, months, days, hours, mins, secs + ) + + +@overload +def make_timestamp_ntz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... + + +@overload +def make_timestamp_ntz( + *, + date: "ColumnOrName", + time: "ColumnOrName", +) -> Column: ... @_try_remote_functions -def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: +def make_timestamp_ntz( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Trim the spaces from both ends for the specified string column. + Create local date-time from years, months, days, hours, mins, secs fields. Alternatively, try to + create local date-time from date and time fields. If the configuration `spark.sql.ansi.enabled` + is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error. - .. versionadded:: 1.5.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space. - A column that evaluates to a string. - - .. versionadded:: 4.0.0 + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - trimmed values from both sides. - Returns a column that evaluates to a string. + A new column that contains a local date-time. + Returns a column that evaluates to a timestamp. See Also -------- - :meth:`pyspark.sql.functions.ltrim` - :meth:`pyspark.sql.functions.rtrim` + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - Example 1: Trim the spaces - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") - >>> df.select("*", sf.trim("value")).show() - +--------+-----------+ - | value|trim(value)| - +--------+-----------+ - | Spark| Spark| - | Spark | Spark| - | Spark| Spark| - +--------+-----------+ + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - Example 2: Trim specified characters + Example 1: Make local date-time from years, months, days, hours, mins, secs. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") - >>> df.select("*", sf.trim("value", sf.lit("*"))).show() - +--------+-----------------------+ - | value|TRIM(BOTH * FROM value)| - +--------+-----------------------+ - |***Spark| Spark| - | Spark**| Spark| - | *Spark| Spark| - +--------+-----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |make_timestamp_ntz(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +----------------------------------------------------+ - Example 3: Trim a column containing different characters + Example 2: Make local date-time from date and time. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) - >>> df.select("*", sf.trim("value", "t")).show() - +--------+---+-----------------------+ - | value| t|TRIM(BOTH t FROM value)| - +--------+---+-----------------------+ - |**Spark*| *| Spark| - |==Spark=| =| Spark| - +--------+---+-----------------------+ + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time") + ... ) + >>> df.select(sf.make_timestamp_ntz(date=df.date, time=df.time)).show(truncate=False) + +------------------------------+ + |make_timestamp_ntz(date, time)| + +------------------------------+ + |2014-12-28 06:30:45.887 | + +------------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - if trim is not None: - return _invoke_function_over_columns("trim", col, trim) + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "make_timestamp_ntz", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) else: - return _invoke_function_over_columns("trim", col) + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) + + +@overload +def try_make_timestamp_ntz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... + + +@overload +def try_make_timestamp_ntz( + *, + date: "ColumnOrName", + time: "ColumnOrName", +) -> Column: ... @_try_remote_functions -def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: +def try_make_timestamp_ntz( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Concatenates multiple input string columns together into a single string column, - using the given separator. + Try to create local date-time from years, months, days, hours, mins, secs fields. Alternatively, + try to create local date-time from date and time fields. The function returns NULL on invalid + inputs. - .. versionadded:: 1.5.0 + .. versionadded:: 4.0.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. Parameters ---------- - sep : literal string - words separator. - A column that evaluates to a string. - cols : :class:`~pyspark.sql.Column` or column name - list of columns to work on. - Each a column that evaluates to a string or an array of strings. + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - string of concatenated words. - Returns a column that evaluates to a string. + A new column that contains a local date-time, or NULL in case of an error. + Returns a column that evaluates to a timestamp. See Also -------- - :meth:`pyspark.sql.functions.concat` + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd", "123")], ["s", "d"]) - >>> df.select("*", sf.concat_ws("-", df.s, "d", sf.lit("xyz"))).show() - +----+---+-----------------------+ - | s| d|concat_ws(-, s, d, xyz)| - +----+---+-----------------------+ - |abcd|123| abcd-123-xyz| - +----+---+-----------------------+ - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - sc = _get_active_spark_context() - return _invoke_function("concat_ws", _enum_to_value(sep), _to_seq(sc, cols, _to_java_column)) + Example 1: Make local date-time from years, months, days, hours, mins, secs. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ntz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +--------------------------------------------------------+ + Example 2: Make local date-time with invalid input -@_try_remote_functions -def decode(col: "ColumnOrName", charset: str) -> Column: + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ntz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |NULL | + +--------------------------------------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - Computes the first argument into a string from a binary using the provided character set - (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "try_make_timestamp_ntz", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) + else: + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "try_make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) - .. versionadded:: 1.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def make_ym_interval( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, +) -> Column: + """ + Make year-month interval from years, months. + + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - charset : literal string - charset to use to decode to. + years : :class:`~pyspark.sql.Column` or column name, optional + The number of years, positive or negative. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The number of months, positive or negative. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + A new column that contains a year-month interval. + Returns a column that evaluates to an interval. See Also -------- - :meth:`pyspark.sql.functions.encode` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.make_dt_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b"\x61\x62\x63\x64",)], ["a"]) - >>> df.select("*", sf.decode("a", "UTF-8")).show() - +-------------+----------------+ - | a|decode(a, UTF-8)| - +-------------+----------------+ - |[61 62 63 64]| abcd| - +-------------+----------------+ + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + + Example 1: Make year-month interval from years, months. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) + >>> df.select('*', sf.make_ym_interval('year', df.month)).show(truncate=False) + +----+-----+-------------------------------+ + |year|month|make_ym_interval(year, month) | + +----+-----+-------------------------------+ + |2014|12 |INTERVAL '2015-0' YEAR TO MONTH| + +----+-----+-------------------------------+ + + Example 2: Make year-month interval from years. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) + >>> df.select('*', sf.make_ym_interval(df.year)).show(truncate=False) + +----+-----+-------------------------------+ + |year|month|make_ym_interval(year, 0) | + +----+-----+-------------------------------+ + |2014|12 |INTERVAL '2014-0' YEAR TO MONTH| + +----+-----+-------------------------------+ + + Example 3: Make empty interval. + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.make_ym_interval()).show(truncate=False) + +----------------------------+ + |make_ym_interval(0, 0) | + +----------------------------+ + |INTERVAL '0-0' YEAR TO MONTH| + +----------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - from pyspark.sql.classic.column import _to_java_column + _years = lit(0) if years is None else years + _months = lit(0) if months is None else months + return _invoke_function_over_columns("make_ym_interval", _years, _months) - return _invoke_function("decode", _to_java_column(col), _enum_to_value(charset)) + +# ---------------------- Hash Functions ---------------------- @_try_remote_functions -def encode(col: "ColumnOrName", charset: str) -> Column: +def crc32(col: "ColumnOrName") -> Column: """ - Computes the first argument into a binary from a string using the provided character set - (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). - - .. versionadded:: 1.5.0 + Calculates the cyclic redundancy check value (CRC32) of a binary column and + returns the value as a bigint. .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -15861,236 +15481,176 @@ def encode(col: "ColumnOrName", charset: str) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - charset : literal string - charset to use to encode. - A column that evaluates to a string. + target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. - Returns a column that evaluates to a binary. + Returns a column that evaluates to a long. - See Also - -------- - :meth:`pyspark.sql.functions.decode` + .. versionadded:: 1.5.0 Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd",)], ["c"]) - >>> df.select("*", sf.encode("c", "UTF-8")).show() - +----+----------------+ - | c|encode(c, UTF-8)| - +----+----------------+ - |abcd| [61 62 63 64]| - +----+----------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ABC',)], ['a']) + >>> df.select('*', sf.crc32('a')).show(truncate=False) + +---+----------+ + |a |crc32(a) | + +---+----------+ + |ABC|2743272264| + +---+----------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("encode", _to_java_column(col), _enum_to_value(charset)) + return _invoke_function_over_columns("crc32", col) @_try_remote_functions -def is_valid_utf8(str: "ColumnOrName") -> Column: - """ - Returns true if the input is a valid UTF-8 string, otherwise returns false. +def md5(col: "ColumnOrName") -> Column: + """Calculates the MD5 digest and returns the value as a 32 character hex string. - .. versionadded:: 4.0.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - whether the input string is a valid UTF-8 string. - Returns a column that evaluates to a boolean. - - See Also - -------- - :meth:`pyspark.sql.functions.make_valid_utf8` - :meth:`pyspark.sql.functions.validate_utf8` - :meth:`pyspark.sql.functions.try_validate_utf8` + the column for computed results. + Returns a column that evaluates to a string. Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.is_valid_utf8(sf.lit("SparkSQL"))).show() - +-----------------------+ - |is_valid_utf8(SparkSQL)| - +-----------------------+ - | true| - +-----------------------+ + >>> df = spark.createDataFrame([('ABC',)], ['a']) + >>> df.select('*', sf.md5('a')).show(truncate=False) + +---+--------------------------------+ + |a |md5(a) | + +---+--------------------------------+ + |ABC|902fbdd2b1df0c4f70b4a5d23525e932| + +---+--------------------------------+ """ - return _invoke_function_over_columns("is_valid_utf8", str) + return _invoke_function_over_columns("md5", col) @_try_remote_functions -def make_valid_utf8(str: "ColumnOrName") -> Column: - """ - Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the - Unicode replacement character (U+FFFD). +def xxh3_64(col: "ColumnOrName") -> Column: + """Returns a 64-bit hash value of the argument using the XXH3 algorithm. - .. versionadded:: 4.0.0 + Unlike :func:`xxhash64`, which hashes one or more columns structurally, this hashes the raw + bytes of a single value with seed 0, so its result is byte compatible with the reference XXH3. + + .. versionadded:: 4.4.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The target column to hash, which must have string or binary type. Returns ------- :class:`~pyspark.sql.Column` - the valid UTF-8 version of the given input string. - Returns a column that evaluates to a string. + Returns a column that evaluates to a long. See Also -------- - :meth:`pyspark.sql.functions.is_valid_utf8` - :meth:`pyspark.sql.functions.validate_utf8` - :meth:`pyspark.sql.functions.try_validate_utf8` + :meth:`pyspark.sql.functions.xxh3_128` + :meth:`pyspark.sql.functions.xxhash64` Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_valid_utf8(sf.lit("SparkSQL"))).show() - +-------------------------+ - |make_valid_utf8(SparkSQL)| - +-------------------------+ - | SparkSQL| - +-------------------------+ + >>> df = spark.createDataFrame([('Spark',)], ['a']) + >>> df.select(sf.xxh3_64('a').alias('h')).collect() + [Row(h=80997306238743657)] """ - return _invoke_function_over_columns("make_valid_utf8", str) + return _invoke_function_over_columns("xxh3_64", col) @_try_remote_functions -def validate_utf8(str: "ColumnOrName") -> Column: - """ - Returns the input value if it corresponds to a valid UTF-8 string, or emits an error otherwise. +def xxh3_128(col: "ColumnOrName") -> Column: + """Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. - .. versionadded:: 4.0.0 + .. versionadded:: 4.4.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The target column to hash, which must have string or binary type. Returns ------- :class:`~pyspark.sql.Column` - the input string if it is a valid UTF-8 string, error otherwise. Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.is_valid_utf8` - :meth:`pyspark.sql.functions.make_valid_utf8` - :meth:`pyspark.sql.functions.try_validate_utf8` + :meth:`pyspark.sql.functions.xxh3_64` + :meth:`pyspark.sql.functions.md5` Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.validate_utf8(sf.lit("SparkSQL"))).show() - +-----------------------+ - |validate_utf8(SparkSQL)| - +-----------------------+ - | SparkSQL| - +-----------------------+ + >>> df = spark.createDataFrame([('Spark',)], ['a']) + >>> df.select(sf.xxh3_128('a').alias('h')).collect() + [Row(h='7d57dd84c60c86ca1f4e82ab91a12b5e')] """ - return _invoke_function_over_columns("validate_utf8", str) + return _invoke_function_over_columns("xxh3_128", col) @_try_remote_functions -def try_validate_utf8(str: "ColumnOrName") -> Column: - """ - Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. +def sha1(col: "ColumnOrName") -> Column: + """Returns the hex string result of SHA-1. - .. versionadded:: 4.0.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - the input string if it is a valid UTF-8 string, null otherwise. + the column for computed results. Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.is_valid_utf8` - :meth:`pyspark.sql.functions.make_valid_utf8` - :meth:`pyspark.sql.functions.validate_utf8` + :meth:`pyspark.sql.functions.sha` + :meth:`pyspark.sql.functions.sha2` Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.try_validate_utf8(sf.lit("SparkSQL"))).show() - +---------------------------+ - |try_validate_utf8(SparkSQL)| - +---------------------------+ - | SparkSQL| - +---------------------------+ + >>> df = spark.createDataFrame([('ABC',)], ['a']) + >>> df.select('*', sf.sha1('a')).show(truncate=False) + +---+----------------------------------------+ + |a |sha1(a) | + +---+----------------------------------------+ + |ABC|3c01bdbb26f358bab27f267924aa2c9a03fcfdb8| + +---+----------------------------------------+ """ - return _invoke_function_over_columns("try_validate_utf8", str) + return _invoke_function_over_columns("sha1", col) @_try_remote_functions -def normalize(str: "ColumnOrName", form: Optional["ColumnOrName"] = None) -> Column: - """ - Returns the Unicode normalization of ``str`` using the given normalization ``form``, as - defined by Unicode Standard Annex #15. Normalization is backed by Spark's bundled ICU4J - library rather than the JVM's own Unicode data, so results are stable across JVM vendors - and versions. - - .. versionadded:: 4.4.0 - - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - the input string to normalize. - form : :class:`~pyspark.sql.Column` or column name, optional - the normalization form, one of 'NFC', 'NFD', 'NFKC', 'NFKD' (case-insensitive). - If omitted, 'NFC' is used. - - Returns - ------- - :class:`~pyspark.sql.Column` - the normalized string. - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("\ufb01",)], ["s"]) - >>> df.select(sf.normalize(df.s, sf.lit("NFKC"))).show() - +------------------+ - |normalize(s, NFKC)| - +------------------+ - | fi| - +------------------+ - """ - if form is None: - return _invoke_function_over_columns("normalize", str) - else: - return _invoke_function_over_columns("normalize", str, form) - - -@_try_remote_functions -def format_number(col: "ColumnOrName", d: int) -> Column: - """ - Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places - with HALF_EVEN round mode, and returns the result as a string. +def sha2(col: "ColumnOrName", numBits: int) -> Column: + """Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, + and SHA-512). The numBits indicates the desired bit length of the result, which must have a + value of 224, 256, 384, 512, or 0 (which is equivalent to 256). .. versionadded:: 1.5.0 @@ -16100,561 +15660,495 @@ def format_number(col: "ColumnOrName", d: int) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - the column name of the numeric value to be formatted. - A column that evaluates to a numeric. - d : int - the N decimal places. + target column to compute on. + A column that evaluates to a binary. + numBits : int + the desired bit length of the result, which must have a + value of 224, 256, 384, 512, or 0 (which is equivalent to 256). A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - the column of formatted results. + the column for computed results. Returns a column that evaluates to a string. + See Also + -------- + :meth:`pyspark.sql.functions.sha` + :meth:`pyspark.sql.functions.sha1` + Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(5,)], ["a"]) - >>> df.select("*", sf.format_number("a", 4), sf.format_number(df.a, 6)).show() - +---+-------------------+-------------------+ - | a|format_number(a, 4)|format_number(a, 6)| - +---+-------------------+-------------------+ - | 5| 5.0000| 5.000000| - +---+-------------------+-------------------+ + >>> df = spark.createDataFrame([['Alice'], ['Bob']], ['name']) + >>> df.select('*', sf.sha2('name', 256)).show(truncate=False) + +-----+----------------------------------------------------------------+ + |name |sha2(name, 256) | + +-----+----------------------------------------------------------------+ + |Alice|3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043| + |Bob |cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961| + +-----+----------------------------------------------------------------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function("format_number", _to_java_column(col), _enum_to_value(d)) + if numBits not in [0, 224, 256, 384, 512]: + raise PySparkValueError( + errorClass="VALUE_NOT_ALLOWED", + messageParameters={ + "arg_name": "numBits", + "allowed_values": "[0, 224, 256, 384, 512]", + }, + ) + return _invoke_function("sha2", _to_java_column(col), numBits) @_try_remote_functions -def format_string(format: str, *cols: "ColumnOrName") -> Column: - """ - Formats the arguments in printf-style and returns the result as a string column. +def hash(*cols: "ColumnOrName") -> Column: + """Calculates the hash code of given columns, and returns the result as an int column. - .. versionadded:: 1.5.0 + .. versionadded:: 2.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - format : literal string - string that can contain embedded format tags and used as result column's value. - A column that evaluates to a string. cols : :class:`~pyspark.sql.Column` or column name - column names or :class:`~pyspark.sql.Column`\\s to be used in formatting + one or more columns to compute on. Each a column of any type. Returns ------- :class:`~pyspark.sql.Column` - the column of formatted results. - Returns a column that evaluates to a string. + hash value as int column. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.printf` + :meth:`pyspark.sql.functions.xxhash64` Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(5, "hello")], ["a", "b"]) - >>> df.select("*", sf.format_string('%d %s', "a", df.b)).show() - +---+-----+--------------------------+ - | a| b|format_string(%d %s, a, b)| - +---+-----+--------------------------+ - | 5|hello| 5 hello| - +---+-----+--------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) + >>> df.select('*', sf.hash('c1')).show() + +---+---+----------+ + | c1| c2| hash(c1)| + +---+---+----------+ + |ABC|DEF|-757602832| + +---+---+----------+ - sc = _get_active_spark_context() - return _invoke_function( - "format_string", _enum_to_value(format), _to_seq(sc, cols, _to_java_column) - ) + >>> df.select('*', sf.hash('c1', df.c2)).show() + +---+---+------------+ + | c1| c2|hash(c1, c2)| + +---+---+------------+ + |ABC|DEF| 599895104| + +---+---+------------+ + + >>> df.select('*', sf.hash('*')).show() + +---+---+------------+ + | c1| c2|hash(c1, c2)| + +---+---+------------+ + |ABC|DEF| 599895104| + +---+---+------------+ + """ + return _invoke_function_over_seq_of_columns("hash", cols) @_try_remote_functions -def instr( - str: "ColumnOrName", - substr: Union[Column, str], - start: Optional[Union[Column, int]] = None, - occurrence: Optional[Union[Column, int]] = None, -) -> Column: - """ - Locate the position of the specified occurrence of substr column in the given string. - Returns null if either of the arguments are null. +def xxhash64(*cols: "ColumnOrName") -> Column: + """Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, + and returns the result as a long column. The hash computation uses an initial seed of 42. - .. versionadded:: 1.5.0 + .. versionadded:: 3.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.3.0 - Supports optional `start` and `occurrence` parameters. - - Notes - ----- - The position is not zero based, but 1 based index. Returns 0 if substr - could not be found in str. - Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - substr : :class:`~pyspark.sql.Column` or literal string - substring to look for. - A column that evaluates to a string. - - .. versionchanged:: 4.0.0 - `substr` now accepts column. - start : int or :class:`~pyspark.sql.Column`, optional - Starting position (1-based, can be negative for backward search). - If not specified, defaults to 1. - A column that evaluates to an integer. - occurrence : int or :class:`~pyspark.sql.Column`, optional - Which occurrence to locate (must be > 0). Defaults to 1. - A column that evaluates to an integer. + cols : :class:`~pyspark.sql.Column` or column name + one or more columns to compute on. + Each a column of any type. Returns ------- :class:`~pyspark.sql.Column` - location of the substring as integer. - Returns a column that evaluates to an integer. + hash value as long column. + Returns a column that evaluates to a long. See Also -------- - :meth:`pyspark.sql.functions.locate` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.functions.hash` + :meth:`pyspark.sql.functions.xxh3_64` Examples -------- - Example 1: Using a literal string as the 'substring' - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr(df.s, "b")).show() - +----+-----------+ - | s|instr(s, b)| - +----+-----------+ - |abcd| 2| - | xyz| 0| - +----+-----------+ - - Example 2: Using a Column 'substring' - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr("s", sf.lit("abc").substr(0, 2))).show() - +----+---------------------------+ - | s|instr(s, substr(abc, 0, 2))| - +----+---------------------------+ - |abcd| 1| - | xyz| 0| - +----+---------------------------+ - - Example 3: Using start and occurrence parameters - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr("s", "b", 1, 2)).show() - +-----+-----------------+ - | s|instr(s, b, 1, 2)| - +-----+-----------------+ - |aabcd| 0| - | xyz| 0| - +-----+-----------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) + >>> df.select('*', sf.xxhash64('c1')).show() + +---+---+-------------------+ + | c1| c2| xxhash64(c1)| + +---+---+-------------------+ + |ABC|DEF|4105715581806190027| + +---+---+-------------------+ - Example 4: Using start parameter + >>> df.select('*', sf.xxhash64('c1', df.c2)).show() + +---+---+-------------------+ + | c1| c2| xxhash64(c1, c2)| + +---+---+-------------------+ + |ABC|DEF|3233247871021311208| + +---+---+-------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr("s", "a", 2)).show() - +-----+-----------------+ - | s|instr(s, a, 2, 1)| - +-----+-----------------+ - |aabcd| 2| - | xyz| 0| - +-----+-----------------+ + >>> df.select('*', sf.xxhash64('*')).show() + +---+---+-------------------+ + | c1| c2| xxhash64(c1, c2)| + +---+---+-------------------+ + |ABC|DEF|3233247871021311208| + +---+---+-------------------+ """ - if start is None and occurrence is None: - return _invoke_function_over_columns("instr", str, lit(substr)) - elif start is not None and occurrence is None: - start = lit(start) - return _invoke_function_over_columns("instr", str, lit(substr), start) - else: - start = lit(start) if start is not None else lit(1) - occurrence = lit(occurrence) - return _invoke_function_over_columns("instr", str, lit(substr), start, occurrence) + return _invoke_function_over_seq_of_columns("xxhash64", cols) @_try_remote_functions -def overlay( - src: "ColumnOrName", - replace: "ColumnOrName", - pos: Union["ColumnOrName", int], - len: Union["ColumnOrName", int] = -1, -) -> Column: +def sha(col: "ColumnOrName") -> Column: """ - Overlay the specified portion of `src` with `replace`, - starting from byte position `pos` of `src` and proceeding for `len` bytes. - - .. versionadded:: 3.0.0 + Returns a sha1 hash value as a hex string of the `col`. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - src : :class:`~pyspark.sql.Column` or column name - the string that will be replaced. - A column that evaluates to a string or binary. - replace : :class:`~pyspark.sql.Column` or column name - the substitution string. - A column that evaluates to a string or binary. - pos : :class:`~pyspark.sql.Column` or column name or int - the starting position in src. - A column that evaluates to an integer. - len : :class:`~pyspark.sql.Column` or column name or int, optional - the number of bytes to replace in src - string by 'replace' defaults to -1, which represents the length of the 'replace' string. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a binary. - Returns - ------- - :class:`~pyspark.sql.Column` - string with replaced values. - Returns a column of the same type as the input. + See Also + -------- + :meth:`pyspark.sql.functions.sha1` + :meth:`pyspark.sql.functions.sha2` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("SPARK_SQL", "CORE")], ("x", "y")) - >>> df.select("*", sf.overlay("x", df.y, 7)).show() - +---------+----+--------------------+ - | x| y|overlay(x, y, 7, -1)| - +---------+----+--------------------+ - |SPARK_SQL|CORE| SPARK_CORE| - +---------+----+--------------------+ - - >>> df.select("*", sf.overlay("x", df.y, 7, 0)).show() - +---------+----+-------------------+ - | x| y|overlay(x, y, 7, 0)| - +---------+----+-------------------+ - |SPARK_SQL|CORE| SPARK_CORESQL| - +---------+----+-------------------+ - - >>> df.select("*", sf.overlay("x", "y", 7, 2)).show() - +---------+----+-------------------+ - | x| y|overlay(x, y, 7, 2)| - +---------+----+-------------------+ - |SPARK_SQL|CORE| SPARK_COREL| - +---------+----+-------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.sha(sf.lit("Spark"))).show() + +--------------------+ + | sha(Spark)| + +--------------------+ + |85f5955f4b27a9a4c...| + +--------------------+ """ - pos = _enum_to_value(pos) - if not isinstance(pos, (int, str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column, int or str", - "arg_name": "pos", - "arg_type": type(pos).__name__, - }, - ) - len = _enum_to_value(len) - if len is not None and not isinstance(len, (int, str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column, int or str", - "arg_name": "len", - "arg_type": type(len).__name__, - }, - ) + return _invoke_function_over_columns("sha", col) - if isinstance(pos, int): - pos = lit(pos) - if isinstance(len, int): - len = lit(len) - return _invoke_function_over_columns("overlay", src, replace, pos, len) +# ---------------------- Collection Functions ---------------------- @_try_remote_functions -def sentences( - string: "ColumnOrName", - language: Optional["ColumnOrName"] = None, - country: Optional["ColumnOrName"] = None, -) -> Column: +def concat(*cols: "ColumnOrName") -> Column: """ - Splits a string into arrays of sentences, where each sentence is an array of words. - The `language` and `country` arguments are optional, - When they are omitted: - 1.If they are both omitted, the `Locale.ROOT - locale(language='', country='')` is used. - The `Locale.ROOT` is regarded as the base locale of all locales, and is used as the - language/country neutral locale for the locale sensitive operations. - 2.If the `country` is omitted, the `locale(language, country='')` is used. - When they are null: - 1.If they are both `null`, the `Locale.US - locale(language='en', country='US')` is used. - 2.If the `language` is null and the `country` is not null, - the `Locale.US - locale(language='en', country='US')` is used. - 3.If the `language` is not null and the `country` is null, the `locale(language)` is used. - 4.If neither is `null`, the `locale(language, country)` is used. + Collection function: Concatenates multiple input columns together into a single column. + The function works with strings, numeric, binary and compatible array columns. - .. versionadded:: 3.2.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.0.0 - Supports `sentences(string, language)`. - Parameters ---------- - string : :class:`~pyspark.sql.Column` or column name - a string to be split. - A column that evaluates to a string. - language : :class:`~pyspark.sql.Column` or column name, optional - a language of the locale. - A column that evaluates to a string. - country : :class:`~pyspark.sql.Column` or column name, optional - a country of the locale. - A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or str + target column or columns to work on. + Each a column that evaluates to a string, numeric, binary, or array. Returns ------- :class:`~pyspark.sql.Column` - arrays of split sentences. - Returns a column that evaluates to an array. + concatenated values. Type of the `Column` depends on input columns' type. + Returns a column of the same type as the input. See Also -------- - :meth:`pyspark.sql.functions.split` - :meth:`pyspark.sql.functions.split_part` + :meth:`pyspark.sql.functions.concat_ws` + :meth:`pyspark.sql.functions.array_join` : to concatenate string columns with delimiter Examples -------- + Example 1: Concatenating string columns + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("This is an example sentence.", )], ["s"]) - >>> df.select("*", sf.sentences(df.s, sf.lit("en"), sf.lit("US"))).show(truncate=False) - +----------------------------+-----------------------------------+ - |s |sentences(s, en, US) | - +----------------------------+-----------------------------------+ - |This is an example sentence.|[[This, is, an, example, sentence]]| - +----------------------------+-----------------------------------+ + >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd']) + >>> df.select(sf.concat(df.s, df.d)).show() + +------------+ + |concat(s, d)| + +------------+ + | abcd123| + +------------+ - >>> df.select("*", sf.sentences(df.s, sf.lit("en"))).show(truncate=False) - +----------------------------+-----------------------------------+ - |s |sentences(s, en, ) | - +----------------------------+-----------------------------------+ - |This is an example sentence.|[[This, is, an, example, sentence]]| - +----------------------------+-----------------------------------+ + Example 2: Concatenating array columns - >>> df.select("*", sf.sentences(df.s)).show(truncate=False) - +----------------------------+-----------------------------------+ - |s |sentences(s, , ) | - +----------------------------+-----------------------------------+ - |This is an example sentence.|[[This, is, an, example, sentence]]| - +----------------------------+-----------------------------------+ - """ - if language is None: - language = lit("") - if country is None: - country = lit("") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2], [3, 4], [5]), ([1, 2], None, [3])], ['a', 'b', 'c']) + >>> df.select(sf.concat(df.a, df.b, df.c)).show() + +---------------+ + |concat(a, b, c)| + +---------------+ + |[1, 2, 3, 4, 5]| + | NULL| + +---------------+ - return _invoke_function_over_columns("sentences", string, language, country) + Example 3: Concatenating numeric columns + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) + >>> df.select(sf.concat(df.a, df.b, df.c)).show() + +---------------+ + |concat(a, b, c)| + +---------------+ + | 123| + +---------------+ + + Example 4: Concatenating binary columns + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytearray(b'abc'), bytearray(b'def'))], ['a', 'b']) + >>> df.select(sf.concat(df.a, df.b)).show() + +-------------------+ + | concat(a, b)| + +-------------------+ + |[61 62 63 64 65 66]| + +-------------------+ + + Example 5: Concatenating mixed types of columns + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,"abc",3,"def")], ['a','b','c','d']) + >>> df.select(sf.concat(df.a, df.b, df.c, df.d)).show() + +------------------+ + |concat(a, b, c, d)| + +------------------+ + | 1abc3def| + +------------------+ + """ + return _invoke_function_over_seq_of_columns("concat", cols) @_try_remote_functions -def substring( - str: "ColumnOrName", - pos: Union["ColumnOrName", int], - len: Union["ColumnOrName", int], -) -> Column: +def element_at(col: "ColumnOrName", extraction: Any) -> Column: """ - Substring starts at `pos` and is of length `len` when str is String type or - returns the slice of byte array that starts at `pos` in byte and is of length `len` - when str is Binary type. + Collection function: + (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will + throw an error. If index < 0, accesses elements from the last to the first. + If 'spark.sql.ansi.enabled' is set to true, an exception will be thrown if the index is out + of array boundaries instead of returning NULL. - .. versionadded:: 1.5.0 + (map, key) - Returns value for given key in `extraction` if col is map. The function always + returns NULL if the key is not contained in the map. + + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + name of column containing array or map. + A column that evaluates to an array or map. + extraction : + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. + + Returns + ------- + :class:`~pyspark.sql.Column` + value at given position. + Returns a column of the element type of the input array, or the value type of the input map. + Notes ----- The position is not zero based, but 1 based index. - - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string or binary. - pos : :class:`~pyspark.sql.Column` or column name or int - starting position in str. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `pos` now accepts column and column name. - - len : :class:`~pyspark.sql.Column` or column name or int - length of chars. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `len` now accepts column and column name. - - Returns - ------- - :class:`~pyspark.sql.Column` - substring of given value. - Returns a column of the same type as the input. + If extraction is a string, :meth:`element_at` treats it as a literal string, + while :meth:`try_element_at` treats it as a column name. See Also -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.locate` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.Column.substr` + :meth:`pyspark.sql.functions.get` + :meth:`pyspark.sql.functions.try_element_at` Examples -------- - Example 1: Using literal integers as arguments + Example 1: Getting the first element of an array - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('abcd',)], ['s',]) - >>> df.select('*', sf.substring(df.s, 1, 2)).show() - +----+------------------+ - | s|substring(s, 1, 2)| - +----+------------------+ - |abcd| ab| - +----+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.element_at(df.data, 1)).show() + +-------------------+ + |element_at(data, 1)| + +-------------------+ + | a| + +-------------------+ - Example 2: Using columns as arguments + Example 2: Getting the last element of an array using negative index - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) - >>> df.select('*', sf.substring(df.s, 2, df.l)).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, 2, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.element_at(df.data, -1)).show() + +--------------------+ + |element_at(data, -1)| + +--------------------+ + | c| + +--------------------+ - >>> df.select('*', sf.substring(df.s, df.p, 3)).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, p, 3)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + Example 3: Getting a value from a map using a key - >>> df.select('*', sf.substring(df.s, df.p, df.l)).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, p, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.element_at(df.data, sf.lit("a"))).show() + +-------------------+ + |element_at(data, a)| + +-------------------+ + | 1.0| + +-------------------+ - Example 3: Using column names as arguments + Example 4: Getting a non-existing value from a map using a key - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) - >>> df.select('*', sf.substring(df.s, 2, 'l')).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, 2, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.element_at(df.data, sf.lit("c"))).show() + +-------------------+ + |element_at(data, c)| + +-------------------+ + | NULL| + +-------------------+ - >>> df.select('*', sf.substring('s', 'p', 'l')).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, p, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + Example 5: Getting a value from a map using a literal string as the key + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) + >>> df.select(sf.element_at(df.data, 'b')).show() + +-------------------+ + |element_at(data, b)| + +-------------------+ + | 2.0| + +-------------------+ """ - pos = _enum_to_value(pos) - pos = lit(pos) if isinstance(pos, int) else pos - len = _enum_to_value(len) - len = lit(len) if isinstance(len, int) else len - return _invoke_function_over_columns("substring", str, pos, len) + return _invoke_function_over_columns("element_at", col, lit(extraction)) @_try_remote_functions -def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: +def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: """ - Returns the substring from string str before count occurrences of the delimiter delim. - If count is positive, everything the left of the final delimiter (counting from left) is - returned. If count is negative, every to the right of the final delimiter (counting from the - right) is returned. substring_index performs a case-sensitive match when searching for delim. + Collection function: + (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will + throw an error. If index < 0, accesses elements from the last to the first. The function + always returns NULL if the index exceeds the length of the array. - .. versionadded:: 1.5.0 + (map, key) - Returns value for given key. The function always returns NULL if the key is not + contained in the map. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - delim : literal string - delimiter of values. - A column that evaluates to a string. - count : int - number of occurrences. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + name of column containing array or map. + A column that evaluates to an array or map. + extraction : + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. Returns ------- :class:`~pyspark.sql.Column` - substring of given value. - Returns a column that evaluates to a string. + Returns a column of the element type of the input array, or the value type of the input map. + + Notes + ----- + The position is not zero based, but 1 based index. + If extraction is a string, :meth:`try_element_at` treats it as a column name, + while :meth:`element_at` treats it as a literal string. See Also -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.locate` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.Column.substr` + :meth:`pyspark.sql.functions.get` + :meth:`pyspark.sql.functions.element_at` Examples -------- + Example 1: Getting the first element of an array + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a.b.c.d',)], ['s']) - >>> df.select('*', sf.substring_index(df.s, '.', 2)).show() - +-------+------------------------+ - | s|substring_index(s, ., 2)| - +-------+------------------------+ - |a.b.c.d| a.b| - +-------+------------------------+ + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit(1))).show() + +-----------------------+ + |try_element_at(data, 1)| + +-----------------------+ + | a| + +-----------------------+ - >>> df.select('*', sf.substring_index('s', '.', -3)).show() - +-------+-------------------------+ - | s|substring_index(s, ., -3)| - +-------+-------------------------+ - |a.b.c.d| b.c.d| - +-------+-------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + Example 2: Getting the last element of an array using negative index - return _invoke_function( - "substring_index", _to_java_column(str), _enum_to_value(delim), _enum_to_value(count) - ) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit(-1))).show() + +------------------------+ + |try_element_at(data, -1)| + +------------------------+ + | c| + +------------------------+ + + Example 3: Getting a value from a map using a key + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit("a"))).show() + +-----------------------+ + |try_element_at(data, a)| + +-----------------------+ + | 1.0| + +-----------------------+ + + Example 4: Getting a non-existing element from an array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit(4))).show() + +-----------------------+ + |try_element_at(data, 4)| + +-----------------------+ + | NULL| + +-----------------------+ + + Example 5: Getting a non-existing value from a map using a key + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit("c"))).show() + +-----------------------+ + |try_element_at(data, c)| + +-----------------------+ + | NULL| + +-----------------------+ + + Example 6: Getting a value from a map using a column name as the key + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) + >>> df.select(sf.try_element_at(df.data, 'b')).show() + +-----------------------+ + |try_element_at(data, b)| + +-----------------------+ + | 1.0| + +-----------------------+ + """ + return _invoke_function_over_columns("try_element_at", col, extraction) @_try_remote_functions -def levenshtein( - left: "ColumnOrName", right: "ColumnOrName", threshold: Optional[int] = None -) -> Column: - """Computes the Levenshtein distance of the two given strings. +def size(col: "ColumnOrName") -> Column: + """ + Collection function: returns the length of the array or map stored in the column. .. versionadded:: 1.5.0 @@ -16663,1241 +16157,1068 @@ def levenshtein( Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - first column value. - A column that evaluates to a string. - right : :class:`~pyspark.sql.Column` or column name - second column value. - A column that evaluates to a string. - threshold : int, optional - if set when the levenshtein distance of the two given strings - less than or equal to a given threshold then return result distance, or -1. - A column that evaluates to an integer. - - .. versionadded:: 3.5.0 + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - Levenshtein distance as integer value. + length of the array/map. Returns a column that evaluates to an integer. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r']) - >>> df.select('*', sf.levenshtein('l', 'r')).show() - +------+-------+-----------------+ - | l| r|levenshtein(l, r)| - +------+-------+-----------------+ - |kitten|sitting| 3| - +------+-------+-----------------+ - - >>> df.select('*', sf.levenshtein(df.l, df.r, 2)).show() - +------+-------+--------------------+ - | l| r|levenshtein(l, r, 2)| - +------+-------+--------------------+ - |kitten|sitting| -1| - +------+-------+--------------------+ + >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data']) + >>> df.select(size(df.data)).collect() + [Row(size(data)=3), Row(size(data)=1), Row(size(data)=0)] """ - from pyspark.sql.classic.column import _to_java_column - - if threshold is None: - return _invoke_function_over_columns("levenshtein", left, right) - else: - return _invoke_function( - "levenshtein", _to_java_column(left), _to_java_column(right), _enum_to_value(threshold) - ) + return _invoke_function_over_columns("size", col) @_try_remote_functions -def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Computes the Jaro-Winkler similarity between the two given strings. - - The result is a double between 0.0 (no similarity) and 1.0 (identical strings). +def cardinality(col: "ColumnOrName") -> Column: + """ + Collection function: returns the length of the array or map stored in the column. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - first column value. - A column that evaluates to a string. - right : :class:`~pyspark.sql.Column` or column name - second column value. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - Jaro-Winkler similarity as a double value. - Returns a column that evaluates to a double. + length of the array/map. + Returns a column that evaluates to an integer. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('MARTHA', 'MARHTA')], ['l', 'r']) - >>> df.select(sf.jaro_winkler_similarity('l', 'r')).show() - +-----------------------------+ - |jaro_winkler_similarity(l, r)| - +-----------------------------+ - | 0.9611111111111111| - +-----------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [([1, 2, 3],),([1],),([],)], ['data'] + ... ).select(sf.cardinality("data")).show() + +-----------------+ + |cardinality(data)| + +-----------------+ + | 3| + | 1| + | 0| + +-----------------+ """ - return _invoke_function_over_columns("jaro_winkler_similarity", left, right) + return _invoke_function_over_columns("cardinality", col) @_try_remote_functions -def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: +def array_sort( + col: "ColumnOrName", comparator: Optional[Callable[[Column, Column], Column]] = None +) -> Column: """ - Locate the position of the first occurrence of substr in a string column, after position pos. + Collection function: sorts the input array in ascending order. The elements of the input array + must be orderable. Null elements will be placed at the end of the returned array. - .. versionadded:: 1.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Can take a `comparator` function. .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - substr : literal string - a string. - A column that evaluates to a string. - str : :class:`~pyspark.sql.Column` or column name - a Column of :class:`pyspark.sql.types.StringType`. - A column that evaluates to a string. - pos : int, optional - start position (zero based). - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + comparator : callable, optional + A binary ``(Column, Column) -> Column: ...``. + The comparator will take two + arguments representing two elements of the array. It returns a negative integer, 0, or a + positive integer as the first element is less than, equal to, or greater than the second + element. If the comparator function returns null, the function will fail and raise an error. Returns ------- :class:`~pyspark.sql.Column` - position of the substring. - Returns a column that evaluates to an integer. - - Notes - ----- - The position is not zero based, but 1 based index. Returns 0 if substr - could not be found in str. + sorted array. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.Column.substr` + :meth:`pyspark.sql.functions.sort_array` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',)], ['s',]) - >>> df.select('*', sf.locate('b', 's', 1)).show() - +----+---------------+ - | s|locate(b, s, 1)| - +----+---------------+ - |abcd| 2| - +----+---------------+ - - >>> df.select('*', sf.locate('b', df.s, 3)).show() - +----+---------------+ - | s|locate(b, s, 3)| - +----+---------------+ - |abcd| 0| - +----+---------------+ + >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data']) + >>> df.select(array_sort(df.data).alias('r')).collect() + [Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])] + >>> df = spark.createDataFrame([(["foo", "foobar", None, "bar"],),(["foo"],),([],)], ['data']) + >>> df.select(array_sort( + ... "data", + ... lambda x, y: when(x.isNull() | y.isNull(), lit(0)).otherwise(length(y) - length(x)) + ... ).alias("r")).collect() + [Row(r=['foobar', 'foo', None, 'bar']), Row(r=['foo']), Row(r=[])] """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "locate", _enum_to_value(substr), _to_java_column(str), _enum_to_value(pos) - ) + if comparator is None: + return _invoke_function_over_columns("array_sort", col) + else: + return _invoke_higher_order_function("array_sort", [col], [comparator]) @_try_remote_functions -def lpad( - col: "ColumnOrName", - len: Union[Column, int], - pad: Union[Column, str], -) -> Column: +def reverse(col: "ColumnOrName") -> Column: """ - Left-pad the string column to width `len` with `pad`. + Collection function: returns a reversed string, a binary value with bytes in reverse order, + or an array with elements in reverse order. .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.2.0 + Added support for binary type. + Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string or binary. - len : :class:`~pyspark.sql.Column` or int - length of the final string. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. - - pad : :class:`~pyspark.sql.Column` or literal string - chars to prepend. - A column that evaluates to a string or binary. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the element to be reversed. + A column that evaluates to a string, binary, or array. Returns ------- :class:`~pyspark.sql.Column` - left padded result. + A new column that contains a reversed string, a binary value with bytes in reverse order, + or an array with elements in reverse order. Returns a column of the same type as the input. - See Also - -------- - :meth:`pyspark.sql.functions.rpad` - Examples -------- - Example 1: Pad with a literal string + Example 1: Reverse a string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.lpad(df.s, 6, '#')).show() - +----+-------------+ - | s|lpad(s, 6, #)| - +----+-------------+ - |abcd| ##abcd| - | xyz| ###xyz| - | 12| ####12| - +----+-------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark SQL',)], ['data']) + >>> df.select(sf.reverse(df.data)).show() + +-------------+ + |reverse(data)| + +-------------+ + | LQS krapS| + +-------------+ - Example 2: Pad with a bytes column + Example 2: Reverse an array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.lpad(df.s, 6, sf.lit(b"\x75\x76"))).show() - +----+-------------------+ - | s|lpad(s, 6, X'7576')| - +----+-------------------+ - |abcd| uvabcd| - | xyz| uvuxyz| - | 12| uvuv12| - +----+-------------------+ - """ - return _invoke_function_over_columns("lpad", col, lit(len), lit(pad)) + >>> df = spark.createDataFrame([([2, 1, 3],) ,([1],) ,([],)], ['data']) + >>> df.select(sf.reverse(df.data)).show() + +-------------+ + |reverse(data)| + +-------------+ + | [3, 1, 2]| + | [1]| + | []| + +-------------+ + Example 3: Reverse binary data -@_try_remote_functions -def rpad( - col: "ColumnOrName", - len: Union[Column, int], - pad: Union[Column, str], -) -> Column: + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytearray(b"\\xCA\\xFE"),)], "data: binary") + >>> df.select(sf.hex(sf.reverse(df.data))).show() + +------------------+ + |hex(reverse(data))| + +------------------+ + | FECA| + +------------------+ """ - Right-pad the string column to width `len` with `pad`. + return _invoke_function_over_columns("reverse", col) - .. versionadded:: 1.5.0 + +def _unresolved_named_lambda_variable(name: str) -> Column: + """ + Create `o.a.s.sql.expressions.UnresolvedNamedLambdaVariable`, + convert it to o.s.sql.Column and wrap in Python `Column` .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target column to work on. - A column that evaluates to a string or binary. - len : :class:`~pyspark.sql.Column` or int - length of the final string. - A column that evaluates to an integer. + name_parts : str + """ + from py4j.java_gateway import JVMView - .. versionchanged:: 4.0.0 - `pattern` now accepts column. + sc = _get_active_spark_context() + return Column(cast(JVMView, sc._jvm).PythonSQLUtils.unresolvedNamedLambdaVariable(name)) - pad : :class:`~pyspark.sql.Column` or literal string - chars to prepend. - A column that evaluates to a string or binary. - .. versionchanged:: 4.0.0 - `pattern` now accepts column. +def _get_lambda_parameters(f: Callable) -> ValuesView[inspect.Parameter]: + signature = inspect.signature(f) + parameters = signature.parameters.values() - Returns - ------- - :class:`~pyspark.sql.Column` - right padded result. - Returns a column of the same type as the input. + # We should exclude functions that use + # variable args and keyword argnames + # as well as keyword only args + supported_parameter_types = { + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.POSITIONAL_ONLY, + } - See Also - -------- - :meth:`pyspark.sql.functions.lpad` + # Validate that + # function arity is between 1 and 3 + if not (1 <= len(parameters) <= 3): + raise PySparkValueError( + errorClass="WRONG_NUM_ARGS_FOR_HIGHER_ORDER_FUNCTION", + messageParameters={"func_name": f.__name__, "num_args": str(len(parameters))}, + ) - Examples - -------- - Example 1: Pad with a literal string + # and all arguments can be used as positional + if not all(p.kind in supported_parameter_types for p in parameters): + raise PySparkValueError( + errorClass="UNSUPPORTED_PARAM_TYPE_FOR_HIGHER_ORDER_FUNCTION", + messageParameters={"func_name": f.__name__}, + ) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.rpad(df.s, 6, '#')).show() - +----+-------------+ - | s|rpad(s, 6, #)| - +----+-------------+ - |abcd| abcd##| - | xyz| xyz###| - | 12| 12####| - +----+-------------+ + return parameters - Example 2: Pad with a bytes column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.rpad(df.s, 6, sf.lit(b"\x75\x76"))).show() - +----+-------------------+ - | s|rpad(s, 6, X'7576')| - +----+-------------------+ - |abcd| abcduv| - | xyz| xyzuvu| - | 12| 12uvuv| - +----+-------------------+ +def _create_lambda(f: Callable) -> Callable: """ - return _invoke_function_over_columns("rpad", col, lit(len), lit(pad)) - + Create `o.a.s.sql.expressions.LambdaFunction` corresponding + to transformation described by f -@_try_remote_functions -def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: + :param f: A Python of one of the following forms: + - (Column) -> Column: ... + - (Column, Column) -> Column: ... + - (Column, Column, Column) -> Column: ... """ - Repeats a string column n times, and returns it as a new string column. + from py4j.java_gateway import JVMView - .. versionadded:: 1.5.0 + from pyspark.sql.classic.column import _to_seq - .. versionchanged:: 3.4.0 - Supports Spark Connect. + parameters = _get_lambda_parameters(f) - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - n : :class:`~pyspark.sql.Column` or column name or int - number of times to repeat value. - A column that evaluates to an integer. + sc = _get_active_spark_context() - .. versionchanged:: 4.0.0 - `n` now accepts column and column name. + argnames = ["x", "y", "z"] + args = [_unresolved_named_lambda_variable(arg) for arg in argnames[: len(parameters)]] - Returns - ------- - :class:`~pyspark.sql.Column` - string with repeated values. - Returns a column that evaluates to a string. + result = f(*args) - Examples - -------- - Example 1: Repeat with a constant number of times + if not isinstance(result, Column): + raise PySparkValueError( + errorClass="HIGHER_ORDER_FUNCTION_SHOULD_RETURN_COLUMN", + messageParameters={"func_name": f.__name__, "return_type": type(result).__name__}, + ) - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ab',)], ['s',]) - >>> df.select("*", sf.repeat("s", 3)).show() - +---+------------+ - | s|repeat(s, 3)| - +---+------------+ - | ab| ababab| - +---+------------+ + jexpr = result._jc + jargs = _to_seq(sc, [arg._jc for arg in args]) + return cast(JVMView, sc._jvm).PythonSQLUtils.lambdaFunction(jexpr, jargs) - >>> df.select("*", sf.repeat(df.s, sf.lit(4))).show() - +---+------------+ - | s|repeat(s, 4)| - +---+------------+ - | ab| abababab| - +---+------------+ - Example 2: Repeat with a column containing different number of times +def _invoke_higher_order_function( + name: str, + cols: Sequence["ColumnOrName"], + funs: Sequence[Callable], +) -> Column: + """ + Invokes expression identified by name, + (relative to ```org.apache.spark.sql.catalyst.expressions``) + and wraps the result with Column (first Scala one, then Python). - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ab', 5,), ('abc', 6,)], ['s', 't']) - >>> df.select("*", sf.repeat("s", "t")).show() - +---+---+------------------+ - | s| t| repeat(s, t)| - +---+---+------------------+ - | ab| 5| ababababab| - |abc| 6|abcabcabcabcabcabc| - +---+---+------------------+ + :param name: Name of the expression + :param cols: a list of columns + :param funs: a list of (*Column) -> Column functions. + + :return: a Column """ - n = _enum_to_value(n) - n = lit(n) if isinstance(n, int) else n - return _invoke_function_over_columns("repeat", col, n) + from py4j.java_gateway import JVMView + + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + jfuns = [_create_lambda(f) for f in funs] + jcols = [_to_java_column(c) for c in cols] + return Column(cast(JVMView, sc._jvm).PythonSQLUtils.fn(name, _to_seq(sc, jcols + jfuns))) + + +@overload +def transform(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... + + +@overload +def transform(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... @_try_remote_functions -def split( - str: "ColumnOrName", - pattern: Union[Column, str], - limit: Union["ColumnOrName", int] = -1, +def transform( + col: "ColumnOrName", + f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], ) -> Column: """ - Splits str around matches of the given pattern. + Returns an array of elements after applying a transformation to each element in the input array. - .. versionadded:: 1.5.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - a string expression to split. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or literal string - a string representing a regular expression. The regex string should be - a Java regular expression. - A column that evaluates to a string. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. Does not accept column name since string type remain - accepted as a regular expression representation, for backwards compatibility. - In addition to int, `limit` now accepts column and column name. - - limit : :class:`~pyspark.sql.Column` or column name or int - an integer which controls the number of times `pattern` is applied. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + a function that is applied to each element of the input array. + Can take one of the following forms: - * ``limit > 0``: The resulting array's length will not be more than `limit`, and the - resulting array's last entry will contain all input beyond the last - matched pattern. - * ``limit <= 0``: `pattern` will be applied as many times as possible, and the resulting - array can be of any size. + - Unary ``(x: Column) -> Column: ...`` + - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is + a 0-based index of the element. - .. versionchanged:: 3.0 - `split` now takes an optional `limit` field. If not provided, default limit value is -1. + and can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - array of separated strings. + a new array of transformed elements. Returns a column that evaluates to an array. - See Also - -------- - :meth:`pyspark.sql.functions.sentences` - :meth:`pyspark.sql.functions.split_part` - Examples -------- - Example 1: Repeat with a constant pattern - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',]) - >>> df.select('*', sf.split(df.s, '[ABC]')).show() - +--------------+-------------------+ - | s|split(s, [ABC], -1)| - +--------------+-------------------+ - |oneAtwoBthreeC|[one, two, three, ]| - +--------------+-------------------+ - - >>> df.select('*', sf.split(df.s, '[ABC]', 2)).show() - +--------------+------------------+ - | s|split(s, [ABC], 2)| - +--------------+------------------+ - |oneAtwoBthreeC| [one, twoBthreeC]| - +--------------+------------------+ - - >>> df.select('*', sf.split('s', '[ABC]', -2)).show() - +--------------+-------------------+ - | s|split(s, [ABC], -2)| - +--------------+-------------------+ - |oneAtwoBthreeC|[one, two, three, ]| - +--------------+-------------------+ - - Example 2: Repeat with a column containing different patterns and limits - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ('oneAtwoBthreeC', '[ABC]', 2), - ... ('1A2B3C', '[1-9]+', 1), - ... ('aa2bb3cc4', '[1-9]+', -1)], ['s', 'p', 'l']) - >>> df.select('*', sf.split(df.s, df.p)).show() - +--------------+------+---+-------------------+ - | s| p| l| split(s, p, -1)| - +--------------+------+---+-------------------+ - |oneAtwoBthreeC| [ABC]| 2|[one, two, three, ]| - | 1A2B3C|[1-9]+| 1| [, A, B, C]| - | aa2bb3cc4|[1-9]+| -1| [aa, bb, cc, ]| - +--------------+------+---+-------------------+ + >>> df = spark.createDataFrame([(1, [1, 2, 3, 4])], ("key", "values")) + >>> df.select(transform("values", lambda x: x * 2).alias("doubled")).show() + +------------+ + | doubled| + +------------+ + |[2, 4, 6, 8]| + +------------+ - >>> df.select(sf.split('s', df.p, 'l')).show() - +-----------------+ - | split(s, p, l)| - +-----------------+ - |[one, twoBthreeC]| - | [1A2B3C]| - | [aa, bb, cc, ]| - +-----------------+ + >>> def alternate(x, i): + ... return when(i % 2 == 0, x).otherwise(-x) + ... + >>> df.select(transform("values", alternate).alias("alternated")).show() + +--------------+ + | alternated| + +--------------+ + |[1, -2, 3, -4]| + +--------------+ """ - limit = _enum_to_value(limit) - limit = lit(limit) if isinstance(limit, int) else limit - return _invoke_function_over_columns("split", str, lit(pattern), limit) + return _invoke_higher_order_function("transform", [col], [f]) @_try_remote_functions -def rlike(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. +def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: + """ + Returns whether a predicate holds for one or more elements in the array. - .. versionadded:: 3.5.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + ``(x: Column) -> Column: ...`` returning the Boolean expression. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - true if `str` matches a Java regex, or false otherwise. + True if "any" element of an array evaluates to True when passed as an argument to + given function and False otherwise. Returns a column that evaluates to a boolean. - See Also - -------- - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.regexp_like` - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.ilike` - Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"(\d+)")], ["str", "regexp"]) - >>> df.select('*', sf.rlike('str', sf.lit(r'(\d+)'))).show() - +---------+------+-----------------+ - | str|regexp|RLIKE(str, (\d+))| - +---------+------+-----------------+ - |1a 2b 14m| (\d+)| true| - +---------+------+-----------------+ - - >>> df.select('*', sf.rlike('str', sf.lit(r'\d{2}b'))).show() - +---------+------+------------------+ - | str|regexp|RLIKE(str, \d{2}b)| - +---------+------+------------------+ - |1a 2b 14m| (\d+)| false| - +---------+------+------------------+ - - >>> df.select('*', sf.rlike("str", sf.col("regexp"))).show() - +---------+------+------------------+ - | str|regexp|RLIKE(str, regexp)| - +---------+------+------------------+ - |1a 2b 14m| (\d+)| true| - +---------+------+------------------+ - - >>> df.select('*', sf.rlike("str", "regexp")).show() - +---------+------+------------------+ - | str|regexp|RLIKE(str, regexp)| - +---------+------+------------------+ - |1a 2b 14m| (\d+)| true| - +---------+------+------------------+ + >>> df = spark.createDataFrame([(1, [1, 2, 3, 4]), (2, [3, -1, 0])],("key", "values")) + >>> df.select(exists("values", lambda x: x < 0).alias("any_negative")).show() + +------------+ + |any_negative| + +------------+ + | false| + | true| + +------------+ """ - return _invoke_function_over_columns("rlike", str, regexp) + return _invoke_higher_order_function("exists", [col], [f]) @_try_remote_functions -def regexp(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. +def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: + """ + Returns whether a predicate holds for every element in the array. - .. versionadded:: 3.5.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or str - regex pattern to apply. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + ``(x: Column) -> Column: ...`` returning the Boolean expression. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - true if `str` matches a Java regex, or false otherwise. + True if "all" elements of an array evaluates to True when passed as an argument to + given function and False otherwise. Returns a column that evaluates to a boolean. - See Also - -------- - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp_like` - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.ilike` - Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp('str', sf.lit(r'(\d+)'))).show() - +------------------+ - |REGEXP(str, (\d+))| - +------------------+ - | true| - +------------------+ + >>> df = spark.createDataFrame( + ... [(1, ["bar"]), (2, ["foo", "bar"]), (3, ["foobar", "foo"])], + ... ("key", "values") + ... ) + >>> df.select(forall("values", lambda x: x.rlike("foo")).alias("all_foo")).show() + +-------+ + |all_foo| + +-------+ + | false| + | false| + | true| + +-------+ + """ + return _invoke_higher_order_function("forall", [col], [f]) - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp('str', sf.lit(r'\d{2}b'))).show() - +-------------------+ - |REGEXP(str, \d{2}b)| - +-------------------+ - | false| - +-------------------+ - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp('str', sf.col("regexp"))).show() - +-------------------+ - |REGEXP(str, regexp)| - +-------------------+ - | true| - +-------------------+ - """ - return _invoke_function_over_columns("regexp", str, regexp) +@overload +def filter(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... + + +@overload +def filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... @_try_remote_functions -def regexp_like(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. +def filter( + col: "ColumnOrName", + f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], +) -> Column: + """ + Returns an array of elements for which a predicate holds in a given array. - .. versionadded:: 3.5.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or str - regex pattern to apply. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + A function that returns the Boolean expression. + Can take one of the following forms: - Returns - ------- - :class:`~pyspark.sql.Column` - true if `str` matches a Java regex, or false otherwise. - Returns a column that evaluates to a boolean. + - Unary ``(x: Column) -> Column: ...`` + - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is + a 0-based index of the element. - See Also - -------- - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.ilike` + and can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). + + Returns + ------- + :class:`~pyspark.sql.Column` + filtered array of elements where given function evaluated to True + when passed as an argument. + Returns a column that evaluates to an array. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp_like('str', sf.lit(r'(\d+)'))).show() - +-----------------------+ - |REGEXP_LIKE(str, (\d+))| - +-----------------------+ - | true| - +-----------------------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp_like('str', sf.lit(r'\d{2}b'))).show() - +------------------------+ - |REGEXP_LIKE(str, \d{2}b)| - +------------------------+ - | false| - +------------------------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp_like('str', sf.col("regexp"))).show() + >>> df = spark.createDataFrame( + ... [(1, ["2018-09-20", "2019-02-03", "2019-07-01", "2020-06-01"])], + ... ("key", "values") + ... ) + >>> def after_second_quarter(x): + ... return month(to_date(x)) > 6 + ... + >>> df.select( + ... filter("values", after_second_quarter).alias("after_second_quarter") + ... ).show(truncate=False) +------------------------+ - |REGEXP_LIKE(str, regexp)| + |after_second_quarter | +------------------------+ - | true| + |[2018-09-20, 2019-07-01]| +------------------------+ """ - return _invoke_function_over_columns("regexp_like", str, regexp) + return _invoke_higher_order_function("filter", [col], [f]) @_try_remote_functions -def randstr(length: Union[Column, int], seed: Optional[Union[Column, int]] = None) -> Column: - """Returns a string of the specified length whose characters are chosen uniformly at random from - the following pool of characters: 0-9, a-z, A-Z. The random seed is optional. The string length - must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). +def aggregate( + col: "ColumnOrName", + initialValue: "ColumnOrName", + merge: Callable[[Column, Column], Column], + finish: Optional[Callable[[Column], Column]] = None, +) -> Column: + """ + Applies a binary operator to an initial state and all elements in the array, + and reduces this to a single state. The final state is converted into the final result + by applying a finish function. - .. versionadded:: 4.0.0 + Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). + + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - length : :class:`~pyspark.sql.Column` or int - Number of characters in the string to generate. - A column that evaluates to an integer. Must be a constant. - seed : :class:`~pyspark.sql.Column` or int - Optional random number seed to use. - A column that evaluates to an integer or long. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + initialValue : :class:`~pyspark.sql.Column` or str + initial value. Name of column or expression. + A column of any type. + merge : function + a binary function ``(acc: Column, x: Column) -> Column...`` returning expression + of the same type as ``initialValue``. + finish : function, optional + an optional unary function ``(x: Column) -> Column: ...`` + used to convert accumulated value. Returns ------- :class:`~pyspark.sql.Column` - The generated random string with the specified length. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.rand` - :meth:`pyspark.sql.functions.randn` + final value after aggregate function is applied. + Returns a column of the same type as ``initialValue``. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(0, 10, 1, 1).select(sf.randstr(16, 3)).show() - +----------------+ - | randstr(16, 3)| - +----------------+ - |nurJIpH4cmmMnsCG| - |fl9YtT5m01trZtIt| - |PD19rAgscTHS7qQZ| - |2CuAICF5UJOruVv4| - |kNZEs8nDpJEoz3Rl| - |OXiU0KN5eaXfjXFs| - |qfnTM1BZAHtN0gBV| - |1p8XiSKwg33KnRPK| - |od5y5MucayQq1bKK| - |tklYPmKmc5sIppWM| - +----------------+ + >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) + >>> df.select(aggregate("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() + +----+ + | sum| + +----+ + |42.0| + +----+ + + >>> def merge(acc, x): + ... count = acc.count + 1 + ... sum = acc.sum + x + ... return struct(count.alias("count"), sum.alias("sum")) + ... + >>> df.select( + ... aggregate( + ... "values", + ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), + ... merge, + ... lambda acc: acc.sum / acc.count, + ... ).alias("mean") + ... ).show() + +----+ + |mean| + +----+ + | 8.4| + +----+ """ - length = _enum_to_value(length) - length = lit(length) - if seed is None: - return _invoke_function_over_columns("randstr", length) + if finish is not None: + return _invoke_higher_order_function("aggregate", [col, initialValue], [merge, finish]) + else: - seed = _enum_to_value(seed) - seed = lit(seed) - return _invoke_function_over_columns("randstr", length, seed) + return _invoke_higher_order_function("aggregate", [col, initialValue], [merge]) @_try_remote_functions -def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns a count of the number of times that the Java regex pattern `regexp` is matched - in the string `str`. +def reduce( + col: "ColumnOrName", + initialValue: "ColumnOrName", + merge: Callable[[Column, Column], Column], + finish: Optional[Callable[[Column], Column]] = None, +) -> Column: + """ + Applies a binary operator to an initial state and all elements in the array, + and reduces this to a single state. The final state is converted into the final result + by applying a finish function. + + Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + initialValue : :class:`~pyspark.sql.Column` or str + initial value. Name of column or expression. + A column of any type. + merge : function + a binary function ``(acc: Column, x: Column) -> Column...`` returning expression + of the same type as ``zero``. + finish : function, optional + an optional unary function ``(x: Column) -> Column: ...`` + used to convert accumulated value. Returns ------- :class:`~pyspark.sql.Column` - the number of times that a Java regex pattern is matched in the string. - Returns a column that evaluates to an integer. + final value after aggregate function is applied. + Returns a column of the same type as ``initialValue``. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) - >>> df.select('*', sf.regexp_count('str', sf.lit(r'\d+'))).show() - +---------+------+----------------------+ - | str|regexp|regexp_count(str, \d+)| - +---------+------+----------------------+ - |1a 2b 14m| \d+| 3| - +---------+------+----------------------+ - - >>> df.select('*', sf.regexp_count('str', sf.lit(r'mmm'))).show() - +---------+------+----------------------+ - | str|regexp|regexp_count(str, mmm)| - +---------+------+----------------------+ - |1a 2b 14m| \d+| 0| - +---------+------+----------------------+ - - >>> df.select('*', sf.regexp_count("str", sf.col("regexp"))).show() - +---------+------+-------------------------+ - | str|regexp|regexp_count(str, regexp)| - +---------+------+-------------------------+ - |1a 2b 14m| \d+| 3| - +---------+------+-------------------------+ + >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) + >>> df.select(reduce("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() + +----+ + | sum| + +----+ + |42.0| + +----+ - >>> df.select('*', sf.regexp_count(sf.col('str'), "regexp")).show() - +---------+------+-------------------------+ - | str|regexp|regexp_count(str, regexp)| - +---------+------+-------------------------+ - |1a 2b 14m| \d+| 3| - +---------+------+-------------------------+ + >>> def merge(acc, x): + ... count = acc.count + 1 + ... sum = acc.sum + x + ... return struct(count.alias("count"), sum.alias("sum")) + ... + >>> df.select( + ... reduce( + ... "values", + ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), + ... merge, + ... lambda acc: acc.sum / acc.count, + ... ).alias("mean") + ... ).show() + +----+ + |mean| + +----+ + | 8.4| + +----+ """ - return _invoke_function_over_columns("regexp_count", str, regexp) + if finish is not None: + return _invoke_higher_order_function("reduce", [col, initialValue], [merge, finish]) + + else: + return _invoke_higher_order_function("reduce", [col, initialValue], [merge]) @_try_remote_functions -def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: - r"""Extract a specific group matched by the Java regex `regexp`, from the specified string column. - If the regex did not match, or the specified group did not match, an empty string is returned. +def zip_with( + left: "ColumnOrName", + right: "ColumnOrName", + f: Callable[[Column, Column], Column], +) -> Column: + """ + Merge two given arrays, element-wise, into a single array using a function. + If one array is shorter, nulls are appended at the end to match the length of the longer + array, before applying the function. - .. versionadded:: 1.5.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - pattern : str - regex pattern to apply. - A column that evaluates to a string. - idx : int - matched group id. - A column that evaluates to an integer. + left : :class:`~pyspark.sql.Column` or str + name of the first column or expression. + A column that evaluates to an array. + right : :class:`~pyspark.sql.Column` or str + name of the second column or expression. + A column that evaluates to an array. + f : function + a binary function ``(x1: Column, x2: Column) -> Column...`` + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - matched value specified by `idx` group id. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.regexp_extract_all` + array of calculated values derived by applying given function to each pair of arguments. + Returns a column that evaluates to an array. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('100-200',)], ['str']) - >>> df.select('*', sf.regexp_extract('str', r'(\d+)-(\d+)', 1)).show() - +-------+-----------------------------------+ - | str|regexp_extract(str, (\d+)-(\d+), 1)| - +-------+-----------------------------------+ - |100-200| 100| - +-------+-----------------------------------+ - - >>> df = spark.createDataFrame([('foo',)], ['str']) - >>> df.select('*', sf.regexp_extract('str', r'(\d+)', 1)).show() - +---+-----------------------------+ - |str|regexp_extract(str, (\d+), 1)| - +---+-----------------------------+ - |foo| | - +---+-----------------------------+ + >>> df = spark.createDataFrame([(1, [1, 3, 5, 8], [0, 2, 4, 6])], ("id", "xs", "ys")) + >>> df.select(zip_with("xs", "ys", lambda x, y: x ** y).alias("powers")).show(truncate=False) + +---------------------------+ + |powers | + +---------------------------+ + |[1.0, 9.0, 625.0, 262144.0]| + +---------------------------+ - >>> df = spark.createDataFrame([('aaaac',)], ['str']) - >>> df.select('*', sf.regexp_extract(sf.col('str'), '(a+)(b)?(c)', 2)).show() - +-----+-----------------------------------+ - | str|regexp_extract(str, (a+)(b)?(c), 2)| - +-----+-----------------------------------+ - |aaaac| | - +-----+-----------------------------------+ + >>> df = spark.createDataFrame([(1, ["foo", "bar"], [1, 2, 3])], ("id", "xs", "ys")) + >>> df.select(zip_with("xs", "ys", lambda x, y: concat_ws("_", x, y)).alias("xs_ys")).show() + +-----------------+ + | xs_ys| + +-----------------+ + |[foo_1, bar_2, 3]| + +-----------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "regexp_extract", _to_java_column(str), _enum_to_value(pattern), _enum_to_value(idx) - ) + return _invoke_higher_order_function("zip_with", [left, right], [f]) @_try_remote_functions -def regexp_extract_all( - str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None -) -> Column: - r"""Extract all strings in the `str` that match the Java regex `regexp` - and corresponding to the regex group index. +def transform_keys(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: + """ + Applies a function to every key-value pair in a map and returns + a map with the results of those applications as the new keys for the pairs. - .. versionadded:: 3.5.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. - idx : :class:`~pyspark.sql.Column` or int, optional - matched group id. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + name of column or expression + f : function + a binary function ``(k: Column, v: Column) -> Column...`` + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - all strings in the `str` that match a Java regex and corresponding to the regex group index. - Returns a column that evaluates to an array. - - See Also - -------- - :meth:`pyspark.sql.functions.regexp_extract` + a new map of entries where new keys were calculated by applying given function to + each key value argument. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("100-200, 300-400", r"(\d+)-(\d+)")], ["str", "regexp"]) - >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'))).show() - +----------------+-----------+---------------------------------------+ - | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| - +----------------+-----------+---------------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+---------------------------------------+ - - >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), sf.lit(1))).show() - +----------------+-----------+---------------------------------------+ - | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| - +----------------+-----------+---------------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+---------------------------------------+ - - >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), 2)).show() - +----------------+-----------+---------------------------------------+ - | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 2)| - +----------------+-----------+---------------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [200, 400]| - +----------------+-----------+---------------------------------------+ - - >>> df.select('*', sf.regexp_extract_all('str', sf.col("regexp"))).show() - +----------------+-----------+----------------------------------+ - | str| regexp|regexp_extract_all(str, regexp, 1)| - +----------------+-----------+----------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+----------------------------------+ - - >>> df.select('*', sf.regexp_extract_all(sf.col('str'), "regexp")).show() - +----------------+-----------+----------------------------------+ - | str| regexp|regexp_extract_all(str, regexp, 1)| - +----------------+-----------+----------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+----------------------------------+ + >>> df = spark.createDataFrame([(1, {"foo": -2.0, "bar": 2.0})], ("id", "data")) + >>> row = df.select(transform_keys( + ... "data", lambda k, _: upper(k)).alias("data_upper") + ... ).head() + >>> sorted(row["data_upper"].items()) + [('BAR', 2.0), ('FOO', -2.0)] """ - if idx is None: - return _invoke_function_over_columns("regexp_extract_all", str, regexp) - else: - return _invoke_function_over_columns("regexp_extract_all", str, regexp, lit(idx)) + return _invoke_higher_order_function("transform_keys", [col], [f]) @_try_remote_functions -def regexp_replace( - string: "ColumnOrName", - pattern: Union[str, Column], - replacement: Union[str, Column], - position: Optional[Union[int, Column]] = None, -) -> Column: - r"""Replace all substrings of the specified string value that match regexp with replacement. +def transform_values(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: + """ + Applies a function to every key-value pair in a map and returns + a map with the results of those applications as the new values for the pairs. - .. versionadded:: 1.5.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.3.0 - Supports the `position` parameter. Parameters ---------- - string : :class:`~pyspark.sql.Column` or str - column name or column containing the string value. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or str - column object or str containing the regexp pattern. - A column that evaluates to a string. - replacement : :class:`~pyspark.sql.Column` or str - column object or str containing the replacement. - A column that evaluates to a string. - position : :class:`~pyspark.sql.Column` or int, optional - position to start replacement. The first position is 1. - A column that evaluates to an integer. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + name of column or expression + f : function + a binary function ``(k: Column, v: Column) -> Column...`` + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - string with all substrings replaced. - Returns a column that evaluates to a string. + a new map of entries where new values were calculated by applying given function to + each key value argument. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("100-200", r"(\d+)", "--")], - ... ["str", "pattern", "replacement"] - ... ) + >>> df = spark.createDataFrame([(1, {"IT": 10.0, "SALES": 2.0, "OPS": 24.0})], ("id", "data")) + >>> row = df.select(transform_values( + ... "data", lambda k, v: when(k.isin("IT", "OPS"), v + 10.0).otherwise(v) + ... ).alias("new_data")).head() + >>> sorted(row["new_data"].items()) + [('IT', 20.0), ('OPS', 34.0), ('SALES', 2.0)] + """ + return _invoke_higher_order_function("transform_values", [col], [f]) - Example 1: Replaces all the substrings in the `str` column name that - match the regex pattern `(\d+)` (one or more digits) with the replacement - string "--". - >>> df.select('*', sf.regexp_replace('str', r'(\d+)', '--')).show() - +-------+-------+-----------+---------------------------------+ - | str|pattern|replacement|regexp_replace(str, (\d+), --, 1)| - +-------+-------+-----------+---------------------------------+ - |100-200| (\d+)| --| -----| - +-------+-------+-----------+---------------------------------+ +@_try_remote_functions +def map_filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: + """ + Collection function: Returns a new map column whose key-value pairs satisfy a given + predicate function. - Example 2: Replaces all the substrings in the `str` Column that match - the regex pattern in the `pattern` Column with the string in the `replacement` - column. + .. versionadded:: 3.1.0 - >>> df.select('*', \ - ... sf.regexp_replace(sf.col("str"), sf.col("pattern"), sf.col("replacement")) \ - ... ).show() - +-------+-------+-----------+--------------------------------------------+ - | str|pattern|replacement|regexp_replace(str, pattern, replacement, 1)| - +-------+-------+-----------+--------------------------------------------+ - |100-200| (\d+)| --| -----| - +-------+-------+-----------+--------------------------------------------+ - - Example 3: Replaces substrings starting from the specified position. - For the input string "100-200", position 5 starts replacement after "100-". - - >>> df.select(sf.regexp_replace("str", r"(\d+)", "--", 5).alias("d")).show() - +------+ - | d| - +------+ - |100---| - +------+ - """ - if position is None: - return _invoke_function_over_columns( - "regexp_replace", string, lit(pattern), lit(replacement) - ) - else: - return _invoke_function_over_columns( - "regexp_replace", - string, - lit(pattern), - lit(replacement), - lit(position), - ) - - -@_try_remote_functions -def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns the first substring that matches the Java regex `regexp` within the string `str`. - If the regular expression is not found, the result is null. - - .. versionadded:: 3.5.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + The name of the column or a column expression representing the map to be filtered. + f : function + A binary function ``(k: Column, v: Column) -> Column...`` that defines the predicate. + This function should return a boolean column that will be used to filter the input map. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - the first substring that matches a Java regex within the string `str`. - Returns a column that evaluates to a string. + A new map column containing only the key-value pairs that satisfy the predicate. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) - - Example 1: Returns the first substring in the `str` column name that - matches the regex pattern `(\d+)` (one or more digits). - - >>> df.select('*', sf.regexp_substr('str', sf.lit(r'\d+'))).show() - +---------+------+-----------------------+ - | str|regexp|regexp_substr(str, \d+)| - +---------+------+-----------------------+ - |1a 2b 14m| \d+| 1| - +---------+------+-----------------------+ - - Example 2: Returns the first substring in the `str` column name that - matches the regex pattern `(mmm)` (three consecutive 'm' characters) + Example 1: Filtering a map with a simple condition - >>> df.select('*', sf.regexp_substr('str', sf.lit(r'mmm'))).show() - +---------+------+-----------------------+ - | str|regexp|regexp_substr(str, mmm)| - +---------+------+-----------------------+ - |1a 2b 14m| \d+| NULL| - +---------+------+-----------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) + >>> row = df.select( + ... sf.map_filter("data", lambda _, v: v > 30.0).alias("data_filtered") + ... ).head() + >>> sorted(row["data_filtered"].items()) + [('baz', 32.0), ('foo', 42.0)] - Example 3: Returns the first substring in the `str` column name that - matches the regex pattern in `regexp` Column. + Example 2: Filtering a map with a condition on keys - >>> df.select('*', sf.regexp_substr("str", sf.col("regexp"))).show() - +---------+------+--------------------------+ - | str|regexp|regexp_substr(str, regexp)| - +---------+------+--------------------------+ - |1a 2b 14m| \d+| 1| - +---------+------+--------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) + >>> row = df.select( + ... sf.map_filter("data", lambda k, _: k.startswith("b")).alias("data_filtered") + ... ).head() + >>> sorted(row["data_filtered"].items()) + [('bar', 1.0), ('baz', 32.0)] - Example 4: Returns the first substring in the `str` Column that - matches the regex pattern in `regexp` column name. + Example 3: Filtering a map with a complex condition - >>> df.select('*', sf.regexp_substr(sf.col("str"), "regexp")).show() - +---------+------+--------------------------+ - | str|regexp|regexp_substr(str, regexp)| - +---------+------+--------------------------+ - |1a 2b 14m| \d+| 1| - +---------+------+--------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) + >>> row = df.select( + ... sf.map_filter("data", lambda k, v: k.startswith("b") & (v > 1.0)).alias("data_filtered") + ... ).head() + >>> sorted(row["data_filtered"].items()) + [('baz', 32.0)] """ - return _invoke_function_over_columns("regexp_substr", str, regexp) + return _invoke_higher_order_function("map_filter", [col], [f]) @_try_remote_functions -def regexp_instr( - str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None +def map_zip_with( + col1: "ColumnOrName", + col2: "ColumnOrName", + f: Callable[[Column, Column, Column], Column], ) -> Column: - r"""Returns the position of the first substring in the `str` that match the Java regex `regexp` - and corresponding to the regex group index. + """ + Collection: Merges two given maps into a single map by applying a function to + the key-value pairs. - .. versionadded:: 3.5.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. - idx : :class:`~pyspark.sql.Column` or int, optional - matched group id. - A column that evaluates to an integer. + col1 : :class:`~pyspark.sql.Column` or str + The name of the first column or a column expression representing the first map. + col2 : :class:`~pyspark.sql.Column` or str + The name of the second column or a column expression representing the second map. + f : function + A ternary function ``(k: Column, v1: Column, v2: Column) -> Column...`` that defines + how to merge the values from the two maps. This function should return a column that + will be used as the value in the resulting map. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - the position of the first substring in the `str` that match a Java regex and corresponding - to the regex group index. - Returns a column that evaluates to an integer. + A new map column where each key-value pair is the result of applying the function to + the corresponding key-value pairs in the input maps. Examples -------- + Example 1: Merging two maps with a simple function + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+(a|b|m)")], ["str", "regexp"]) + >>> df = spark.createDataFrame([ + ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], + ... ("id", "map1", "map2")) + >>> row = df.select( + ... sf.map_zip_with("map1", "map2", lambda _, v1, v2: v1 + v2).alias("updated_data") + ... ).head() + >>> sorted(row["updated_data"].items()) + [('A', 4), ('B', 6)] - Example 1: Returns the position of the first substring in the `str` column name that - match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'). + Example 2: Merging two maps with a complex function - >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'))).show() - +---------+----------+--------------------------------+ - | str| regexp|regexp_instr(str, \d+(a|b|m), 0)| - +---------+----------+--------------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+--------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], + ... ("id", "map1", "map2")) + >>> row = df.select( + ... sf.map_zip_with("map1", "map2", + ... lambda k, v1, v2: sf.when(k == "A", v1 + v2).otherwise(v1 - v2) + ... ).alias("updated_data") + ... ).head() + >>> sorted(row["updated_data"].items()) + [('A', 4), ('B', -2)] - Example 2: Returns the position of the first substring in the `str` column name that - match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'), + Example 3: Merging two maps with mismatched keys - >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'), sf.lit(1))).show() - +---------+----------+--------------------------------+ - | str| regexp|regexp_instr(str, \d+(a|b|m), 1)| - +---------+----------+--------------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+--------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (1, {"A": 1, "B": 2}, {"B": 3, "C": 4})], + ... ("id", "map1", "map2")) + >>> row = df.select( + ... sf.map_zip_with("map1", "map2", + ... lambda _, v1, v2: sf.when(v2.isNull(), v1).otherwise(v1 + v2) + ... ).alias("updated_data") + ... ).head() + >>> sorted(row["updated_data"].items()) + [('A', 1), ('B', 5), ('C', None)] + """ + return _invoke_higher_order_function("map_zip_with", [col1, col2], [f]) - Example 3: Returns the position of the first substring in the `str` column name that - match the regex pattern in `regexp` Column. - >>> df.select('*', sf.regexp_instr('str', sf.col("regexp"))).show() - +---------+----------+----------------------------+ - | str| regexp|regexp_instr(str, regexp, 0)| - +---------+----------+----------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+----------------------------+ +# ---------------------- Array Functions ---------------------- - Example 4: Returns the position of the first substring in the `str` Column that - match the regex pattern in `regexp` column name. - >>> df.select('*', sf.regexp_instr(sf.col("str"), "regexp")).show() - +---------+----------+----------------------------+ - | str| regexp|regexp_instr(str, regexp, 0)| - +---------+----------+----------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+----------------------------+ - """ - if idx is None: - return _invoke_function_over_columns("regexp_instr", str, regexp) - else: - return _invoke_function_over_columns("regexp_instr", str, regexp, lit(idx)) +@overload +def array(*cols: "ColumnOrName") -> Column: ... + + +@overload +def array(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... @_try_remote_functions -def initcap(col: "ColumnOrName") -> Column: - """Translate the first letter of each word to upper case in the sentence. +def array( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], +) -> Column: + """ + Collection function: Creates a new array column from the input columns or column names. - .. versionadded:: 1.5.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or str + Column names or :class:`~pyspark.sql.Column` objects that have the same data type. Returns ------- :class:`~pyspark.sql.Column` - string with all first letters are uppercase in each word. - Returns a column that evaluates to a string. + A new Column of array type, where each value is an array containing the corresponding values + from the input columns. + Returns a column that evaluates to an array. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ab cd',)], ['a']) - >>> df.select("*", sf.initcap("a")).show() - +-----+----------+ - | a|initcap(a)| - +-----+----------+ - |ab cd| Ab Cd| - +-----+----------+ - """ - return _invoke_function_over_columns("initcap", col) + Example 1: Basic usage of array function with column names. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array('name', 'occupation')).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, doctor]| + | [Bob, engineer]| + +-----------------------+ -@_try_remote_functions -def soundex(col: "ColumnOrName") -> Column: - """ - Returns the SoundEx encoding for a string + Example 2: Usage of array function with Column objects. - .. versionadded:: 1.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array(df.name, df.occupation)).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, doctor]| + | [Bob, engineer]| + +-----------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Single argument as list of column names. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array(['name', 'occupation'])).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, doctor]| + | [Bob, engineer]| + +-----------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - SoundEx encoded string. - Returns a column that evaluates to a string. + Example 4: Usage of array function with columns of different types. - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Peters",),("Uhrbach",)], ["s"]) - >>> df.select("*", sf.soundex("s")).show() - +-------+----------+ - | s|soundex(s)| - +-------+----------+ - | Peters| P362| - |Uhrbach| U612| - +-------+----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("Alice", 2, 22.2), ("Bob", 5, 36.1)], + ... ("name", "age", "weight")) + >>> df.select(sf.array(['age', 'weight'])).show() + +------------------+ + |array(age, weight)| + +------------------+ + | [2.0, 22.2]| + | [5.0, 36.1]| + +------------------+ + + Example 5: array function with a column containing null values. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", None), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array('name', 'occupation')).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, NULL]| + | [Bob, engineer]| + +-----------------------+ """ - return _invoke_function_over_columns("soundex", col) + if len(cols) == 1 and isinstance(cols[0], (list, set)): + cols = cols[0] # type: ignore[assignment] + return _invoke_function_over_seq_of_columns("array", cols) # type: ignore[arg-type] @_try_remote_functions -def bin(col: "ColumnOrName") -> Column: - """Returns the string representation of the binary value of the given column. +def array_contains(col: "ColumnOrName", value: Any) -> Column: + """ + Collection function: Returns true if the array contains the value, false if not. Returns + null if the array or value is null, or if the value is not found and the array contains a + null element. .. versionadded:: 1.5.0 @@ -17906,2068 +17227,2408 @@ def bin(col: "ColumnOrName") -> Column: Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a long. + col : :class:`~pyspark.sql.Column` or str + The target column containing the arrays. + A column that evaluates to an array. + value : + The value or column to check for in the array. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` - binary representation of given value as string. - Returns a column that evaluates to a string. + A new Column of Boolean type, where each value indicates whether the corresponding array + from the input column contains the specified value. + Returns a column that evaluates to a boolean. + + See Also + -------- + :meth:`pyspark.sql.functions.array_position` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(10).select("*", sf.bin("id")).show() - +---+-------+ - | id|bin(id)| - +---+-------+ - | 0| 0| - | 1| 1| - | 2| 10| - | 3| 11| - | 4| 100| - | 5| 101| - | 6| 110| - | 7| 111| - | 8| 1000| - | 9| 1001| - +---+-------+ - """ - return _invoke_function_over_columns("bin", col) + Example 1: Basic usage of array_contains function. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],), ([],)], ['data']) + >>> df.select(sf.array_contains(df.data, "a")).show() + +-----------------------+ + |array_contains(data, a)| + +-----------------------+ + | true| + | false| + +-----------------------+ -@_try_remote_functions -def hex(col: "ColumnOrName") -> Column: - """Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`, - :class:`pyspark.sql.types.BinaryType`, :class:`pyspark.sql.types.IntegerType` or - :class:`pyspark.sql.types.LongType`. + Example 2: Usage of array_contains function with a column. - .. versionadded:: 1.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"], "c"), + ... (["c", "d", "e"], "d"), + ... (["e", "a", "c"], "b")], ["data", "item"]) + >>> df.select(sf.array_contains(df.data, sf.col("item"))).show() + +--------------------------+ + |array_contains(data, item)| + +--------------------------+ + | true| + | true| + | false| + +--------------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Attempt to use array_contains function with a null array. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a long, binary, or string. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(None,), (["a", "b", "c"],)], ['data']) + >>> df.select(sf.array_contains(df.data, "a")).show() + +-----------------------+ + |array_contains(data, a)| + +-----------------------+ + | NULL| + | true| + +-----------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.unhex` + Example 4: Usage of array_contains with an array column containing null values. - Returns - ------- - :class:`~pyspark.sql.Column` - hexadecimal representation of given value as string. - Returns a column that evaluates to a string. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_contains(df.data, "a")).show() + +-----------------------+ + |array_contains(data, a)| + +-----------------------+ + | true| + +-----------------------+ - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC', 3)], ['a', 'b']) - >>> df.select('*', sf.hex('a'), sf.hex(df.b)).show() - +---+---+------+------+ - | a| b|hex(a)|hex(b)| - +---+---+------+------+ - |ABC| 3|414243| 3| - +---+---+------+------+ + Example 5: Value absent from an array that contains a null element returns NULL. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_contains(df.data, "b")).show() + +-----------------------+ + |array_contains(data, b)| + +-----------------------+ + | NULL| + +-----------------------+ """ - return _invoke_function_over_columns("hex", col) + return _invoke_function_over_columns("array_contains", col, lit(value)) @_try_remote_functions -def unhex(col: "ColumnOrName") -> Column: - """Inverse of hex. Interprets each pair of characters as a hexadecimal number - and converts to the byte representation of number. +def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column: + """ + Collection function: This function returns a boolean column indicating if the input arrays + have common non-null elements, returning true if they do, null if the arrays do not contain + any common elements but are not empty and at least one of them contains a null element, + and false otherwise. - .. versionadded:: 1.5.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.hex` + a1, a2 : :class:`~pyspark.sql.Column` or str + The names of the columns that contain the input arrays. + Each a column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - byte representation of the given hexadecimal value. - Returns a column that evaluates to a binary. + A new Column of Boolean type, where each value indicates whether the corresponding arrays + from the input columns contain any common elements. + Returns a column that evaluates to a boolean. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('414243',)], ['a']) - >>> df.select('*', sf.unhex('a')).show() - +------+----------+ - | a| unhex(a)| - +------+----------+ - |414243|[41 42 43]| - +------+----------+ - """ - return _invoke_function_over_columns("unhex", col) + Example 1: Basic usage of arrays_overlap function. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b"], ["b", "c"]), (["a"], ["b", "c"])], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | true| + | false| + +--------------------+ -@_try_remote_functions -def uniform( - min: Union[Column, int, float], - max: Union[Column, int, float], - seed: Optional[Union[Column, int]] = None, -) -> Column: - """Returns a random value with independent and identically distributed (i.i.d.) values with the - specified range of numbers. The random seed is optional. The provided numbers specifying the - minimum and maximum values of the range must be constant. If both of these numbers are integers, - then the result will also be an integer. Otherwise if one or both of these are floating-point - numbers, then the result will also be a floating-point number. + Example 2: Usage of arrays_overlap function with arrays containing null elements. - .. versionadded:: 4.0.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", None], ["b", None]), (["a"], ["b", "c"])], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | NULL| + | false| + +--------------------+ - Parameters - ---------- - min : :class:`~pyspark.sql.Column`, int, or float - Minimum value in the range. - A column that evaluates to a numeric. Must be a constant. - max : :class:`~pyspark.sql.Column`, int, or float - Maximum value in the range. - A column that evaluates to a numeric. Must be a constant. - seed : :class:`~pyspark.sql.Column` or int - Optional random number seed to use. - A column that evaluates to an integer or long. Must be a constant. + Example 3: Usage of arrays_overlap function with arrays that are null. - Returns - ------- - :class:`~pyspark.sql.Column` - The generated random number within the specified range. - Returns a column of the same type as the input. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(None, ["b", "c"]), (["a"], None)], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | NULL| + | NULL| + +--------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.rand` - :meth:`pyspark.sql.functions.randn` + Example 4: Usage of arrays_overlap on arrays with identical elements. - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(0, 10, 1, 1).select(sf.uniform(5, 105, 3)).show() - +------------------+ - |uniform(5, 105, 3)| - +------------------+ - | 30| - | 71| - | 99| - | 77| - | 16| - | 25| - | 89| - | 80| - | 51| - | 83| - +------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b"], ["a", "b"]), (["a"], ["a"])], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | true| + | true| + +--------------------+ """ - min = _enum_to_value(min) - min = lit(min) - max = _enum_to_value(max) - max = lit(max) - if seed is None: - return _invoke_function_over_columns("uniform", min, max) - else: - seed = _enum_to_value(seed) - seed = lit(seed) - return _invoke_function_over_columns("uniform", min, max, seed) + return _invoke_function_over_columns("arrays_overlap", a1, a2) @_try_remote_functions -def length(col: "ColumnOrName") -> Column: - """Computes the character length of string data or number of bytes of binary data. - The length of character data includes the trailing spaces. The length of binary data - includes binary zeros. +def slice( + x: "ColumnOrName", start: Union["ColumnOrName", int], length: Union["ColumnOrName", int] +) -> Column: + """ + Array function: Returns a new array column by slicing the input array column from + a start index to a specific length. The indices start at 1, and can be negative to index + from the end of the array. The length specifies the number of elements in the resulting array. - .. versionadded:: 1.5.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string or binary. + x : :class:`~pyspark.sql.Column` or str + Input array column or column name to be sliced. + A column that evaluates to an array. + start : :class:`~pyspark.sql.Column`, str, or int + The start index for the slice operation. If negative, starts the index from the + end of the array. + A column that evaluates to an integer. + length : :class:`~pyspark.sql.Column`, str, or int + The length of the slice, representing number of elements in the resulting array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - length of the value. - Returns a column that evaluates to an integer. - - See Also - -------- - :meth:`pyspark.sql.functions.char_length` - :meth:`pyspark.sql.functions.character_length` + A new Column object of Array type, where each value is a slice of the corresponding + list from the input column. + Returns a column that evaluates to an array. Examples -------- + Example 1: Basic usage of the slice function. + >>> from pyspark.sql import functions as sf - >>> spark.createDataFrame([('ABC ',)], ['a']).select('*', sf.length('a')).show() - +----+---------+ - | a|length(a)| - +----+---------+ - |ABC | 4| - +----+---------+ + >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) + >>> df.select(sf.slice(df.x, 2, 2)).show() + +--------------+ + |slice(x, 2, 2)| + +--------------+ + | [2, 3]| + | [5]| + +--------------+ + + Example 2: Slicing with negative start index. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) + >>> df.select(sf.slice(df.x, -1, 1)).show() + +---------------+ + |slice(x, -1, 1)| + +---------------+ + | [3]| + | [5]| + +---------------+ + + Example 3: Slice function with column inputs for start and length. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3], 2, 2), ([4, 5], 1, 3)], ['x', 'start', 'length']) + >>> df.select(sf.slice(df.x, df.start, df.length)).show() + +-----------------------+ + |slice(x, start, length)| + +-----------------------+ + | [2, 3]| + | [4, 5]| + +-----------------------+ """ - return _invoke_function_over_columns("length", col) + start = _enum_to_value(start) + start = lit(start) if isinstance(start, int) else start + length = _enum_to_value(length) + length = lit(length) if isinstance(length, int) else length + + return _invoke_function_over_columns("slice", x, start, length) @_try_remote_functions -def octet_length(col: "ColumnOrName") -> Column: +def trim_array(x: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: """ - Calculates the byte length for the specified string column. - - .. versionadded:: 3.3.0 + Array function: Returns the given array column with the last ``n`` elements removed. + Raises an error if ``n`` is negative or greater than the number of elements in the array. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Source column or strings. - A column that evaluates to a string or binary. + x : :class:`~pyspark.sql.Column` or str + Input array column or column name to be trimmed. + A column that evaluates to an array. + n : :class:`~pyspark.sql.Column`, str, or int + The number of elements to remove from the end of the array. Must be between 0 and + the number of elements in the array (inclusive). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - Byte length of the col - Returns a column that evaluates to an integer. + A new Column object of Array type, where each value is the corresponding input array + with its last ``n`` elements removed. + Returns a column that evaluates to an array. Examples -------- + Example 1: Basic usage of the trim_array function. + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) - >>> df.select('*', sf.octet_length('cat')).show() - +---+-----------------+ - |cat|octet_length(cat)| - +---+-----------------+ - |cat| 3| - | 🐈| 4| - +---+-----------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 4, 5],), ([4, 5],)], ['x']) + >>> df.select(sf.trim_array(df.x, 2)).show() + +----------------+ + |trim_array(x, 2)| + +----------------+ + | [1, 2, 3]| + | []| + +----------------+ + + Example 2: trim_array function with a column input for n. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3, 4, 5], 1), ([4, 5], 0)], ['x', 'n']) + >>> df.select(sf.trim_array(df.x, df.n)).show() + +----------------+ + |trim_array(x, n)| + +----------------+ + | [1, 2, 3, 4]| + | [4, 5]| + +----------------+ """ - return _invoke_function_over_columns("octet_length", col) + n = _enum_to_value(n) + n = lit(n) if isinstance(n, int) else n + return _invoke_function_over_columns("trim_array", x, n) @_try_remote_functions -def bit_length(col: "ColumnOrName") -> Column: +def array_join( + col: "ColumnOrName", delimiter: str, null_replacement: Optional[str] = None +) -> Column: """ - Calculates the bit length for the specified string column. + Array function: Returns a string column by concatenating the elements of the input + array column using the delimiter. Null values within the array can be replaced with + a specified string through the null_replacement argument. If null_replacement is + not set, null values are ignored. - .. versionadded:: 3.3.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Source column or strings. - A column that evaluates to a string or binary. + col : :class:`~pyspark.sql.Column` or str + The input column containing the arrays to be joined. + A column that evaluates to an array. + delimiter : str + The string to be used as the delimiter when joining the array elements. + A column that evaluates to a string. + null_replacement : str, optional + The string to replace null values within the array. If not set, null values are ignored. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - Bit length of the col - Returns a column that evaluates to an integer. + A new column of string type, where each value is the result of joining the corresponding + array from the input column. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.concat` + :meth:`pyspark.sql.functions.concat_ws` Examples -------- + Example 1: Basic usage of array_join function. + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) - >>> df.select('*', sf.bit_length('cat')).show() - +---+---------------+ - |cat|bit_length(cat)| - +---+---------------+ - |cat| 24| - | 🐈| 32| - +---+---------------+ - """ - return _invoke_function_over_columns("bit_length", col) + >>> df = spark.createDataFrame([(["a", "b", "c"],), (["a", "b"],)], ['data']) + >>> df.select(sf.array_join(df.data, ",")).show() + +-------------------+ + |array_join(data, ,)| + +-------------------+ + | a,b,c| + | a,b| + +-------------------+ + Example 2: Usage of array_join function with null_replacement argument. -@_try_remote_functions -def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: - """A function translate any character in the `srcCol` by a character in `matching`. - The characters in `replace` is corresponding to the characters in `matching`. - Translation will happen whenever any character in the string is matching with the character - in the `matching`. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_join(df.data, ",", "NULL")).show() + +-------------------------+ + |array_join(data, ,, NULL)| + +-------------------------+ + | a,NULL,c| + +-------------------------+ - .. versionadded:: 1.5.0 + Example 3: Usage of array_join function without null_replacement argument. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_join(df.data, ",")).show() + +-------------------+ + |array_join(data, ,)| + +-------------------+ + | a,c| + +-------------------+ - Parameters - ---------- - srcCol : :class:`~pyspark.sql.Column` or column name - Source column or strings. - A column that evaluates to a string. - matching : str - matching characters. - A column that evaluates to a string. - replace : str - characters for replacement. If this is shorter than `matching` string then - those chars that don't have replacement will be dropped. - A column that evaluates to a string. + Example 4: Usage of array_join function with an array that is null. - Returns - ------- - :class:`~pyspark.sql.Column` - replaced value. - Returns a column that evaluates to a string. + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([(None,)], schema) + >>> df.select(sf.array_join(df.data, ",")).show() + +-------------------+ + |array_join(data, ,)| + +-------------------+ + | NULL| + +-------------------+ + + Example 5: Usage of array_join function with an array containing only null values. - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('translate',)], ['a']) - >>> df.select('*', sf.translate('a', "rnlt", "123")).show() - +---------+-----------------------+ - | a|translate(a, rnlt, 123)| - +---------+-----------------------+ - |translate| 1a2s3ae| - +---------+-----------------------+ + >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([([None, None],)], schema) + >>> df.select(sf.array_join(df.data, ",", "NULL")).show() + +-------------------------+ + |array_join(data, ,, NULL)| + +-------------------------+ + | NULL,NULL| + +-------------------------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function( - "translate", _to_java_column(srcCol), _enum_to_value(matching), _enum_to_value(replace) - ) + _get_active_spark_context() + if null_replacement is None: + return _invoke_function("array_join", _to_java_column(col), _enum_to_value(delimiter)) + else: + return _invoke_function( + "array_join", + _to_java_column(col), + _enum_to_value(delimiter), + _enum_to_value(null_replacement), + ) @_try_remote_functions -def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: +def array_position(col: "ColumnOrName", value: Any) -> Column: """ - Converts the input `col` to a binary value based on the supplied `format`. - The `format` can be a case-insensitive string literal of "hex", "utf-8", "utf8", - or "base64". By default, the binary format for conversion is "hex" if - `format` is omitted. The function returns NULL if at least one of the - input parameters is NULL. + Array function: Locates the position of the first occurrence of the given value + in the given array. Returns null if either of the arguments are null. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The position is not zero based, but 1 based index. Returns 0 if the given + value could not be found in the array. Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert binary values. - A column that evaluates to a string. Must be a constant. + target column to work on. + A column that evaluates to an array. + value : Any + value or a :class:`~pyspark.sql.Column` expression to look for. + A column of the same type as the array elements. + + .. versionchanged:: 4.0.0 + `value` now also accepts a Column type. + + Returns + ------- + :class:`~pyspark.sql.Column` + position of the value in the given array if found and 0 otherwise. + Returns a column that evaluates to a long. See Also -------- - :meth:`pyspark.sql.functions.try_to_binary` + :meth:`pyspark.sql.functions.array_contains` Examples -------- - Example 1: Convert string to a binary with encoding specified + Example 1: Finding the position of a string in an array of strings - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("abc",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() - [Row(r=b'abc')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) + >>> df.select(sf.array_position(df.data, "a")).show() + +-----------------------+ + |array_position(data, a)| + +-----------------------+ + | 3| + +-----------------------+ - Example 2: Convert string to a timestamp without encoding specified + Example 2: Finding the position of a string in an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_position(df.data, "a")).show() + +-----------------------+ + |array_position(data, a)| + +-----------------------+ + | 0| + +-----------------------+ + + Example 3: Finding the position of an integer in an array of integers + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_position(df.data, 2)).show() + +-----------------------+ + |array_position(data, 2)| + +-----------------------+ + | 2| + +-----------------------+ + + Example 4: Finding the position of a non-existing value in an array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) + >>> df.select(sf.array_position(df.data, "d")).show() + +-----------------------+ + |array_position(data, d)| + +-----------------------+ + | 0| + +-----------------------+ + + Example 5: Finding the position of a value in an array with nulls + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([None, "b", "a"],)], ['data']) + >>> df.select(sf.array_position(df.data, "a")).show() + +-----------------------+ + |array_position(data, a)| + +-----------------------+ + | 3| + +-----------------------+ + + Example 6: Finding the position of a column's value in an array of integers + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([10, 20, 30], 20)], ['data', 'col']) + >>> df.select(sf.array_position(df.data, df.col)).show() + +-------------------------+ + |array_position(data, col)| + +-------------------------+ + | 2| + +-------------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("414243",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() - [Row(r=b'ABC')] """ - if format is not None: - return _invoke_function_over_columns("to_binary", col, format) - else: - return _invoke_function_over_columns("to_binary", col) + return _invoke_function_over_columns("array_position", col, lit(value)) @_try_remote_functions -def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: +def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column: """ - Convert `col` to a string based on the `format`. - Throws an exception if the conversion fails. The format can consist of the following - characters, case insensitive: - '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the - format string matches a sequence of digits in the input value, generating a result - string of the same length as the corresponding sequence in the format string. - The result string is left-padded with zeros if the 0/9 sequence comprises more digits - than the matching part of the decimal value, starts with 0, and is before the decimal - point. Otherwise, it is padded with spaces. - '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). - ',' or 'G': Specifies the position of the grouping (thousands) separator (,). - There must be a 0 or 9 to the left and right of each grouping separator. - '$': Specifies the location of the $ currency sign. This character may only be specified once. - 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at - the beginning or end of the format string). Note that 'S' prints '+' for positive - values but 'MI' prints a space. - 'PR': Only allowed at the end of the format string; specifies that the result string - will be wrapped by angle brackets if the input value is negative. - If `col` is a datetime, `format` shall be a valid datetime pattern, see - Patterns. - If `col` is a binary, it is converted to a string in one of the formats: - 'base64': a base 64 string. - 'hex': a string in the hexadecimal format. - 'utf-8': the input binary is decoded to UTF-8 string. + Array function: Returns the element of an array at the given (0-based) index. + If the index points outside of the array boundaries, then this function + returns NULL. - .. versionadded:: 3.5.0 + .. versionadded:: 3.4.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or str - The value to convert to a string. - A column that evaluates to a numeric, date, timestamp, time, or binary. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert char values. - A column that evaluates to a string. Must be a constant when ``col`` is numeric - or binary. + Name of the column containing the array. + A column that evaluates to an array. + index : :class:`~pyspark.sql.Column` or str or int + Index to check for in the array. + A column that evaluates to an integer. + + Returns + ------- + :class:`~pyspark.sql.Column` + Value at the given position. + Returns a column of the element type of the input array. + + Notes + ----- + The position is not 1-based, but 0-based index. + Supports Spark Connect. + + See Also + -------- + :meth:`pyspark.sql.functions.element_at` + :meth:`pyspark.sql.functions.try_element_at` Examples -------- - >>> df = spark.createDataFrame([(78.12,)], ["e"]) - >>> df.select(to_char(df.e, lit("$99.99")).alias('r')).collect() - [Row(r='$78.12')] - """ - return _invoke_function_over_columns("to_char", col, format) + Example 1: Getting an element at a fixed position + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.get(df.data, 1)).show() + +------------+ + |get(data, 1)| + +------------+ + | b| + +------------+ -@_try_remote_functions -def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: - """ - Convert `col` to a string based on the `format`. - Throws an exception if the conversion fails. The format can consist of the following - characters, case insensitive: - '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the - format string matches a sequence of digits in the input value, generating a result - string of the same length as the corresponding sequence in the format string. - The result string is left-padded with zeros if the 0/9 sequence comprises more digits - than the matching part of the decimal value, starts with 0, and is before the decimal - point. Otherwise, it is padded with spaces. - '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). - ',' or 'G': Specifies the position of the grouping (thousands) separator (,). - There must be a 0 or 9 to the left and right of each grouping separator. - '$': Specifies the location of the $ currency sign. This character may only be specified once. - 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at - the beginning or end of the format string). Note that 'S' prints '+' for positive - values but 'MI' prints a space. - 'PR': Only allowed at the end of the format string; specifies that the result string - will be wrapped by angle brackets if the input value is negative. - If `col` is a datetime, `format` shall be a valid datetime pattern, see - Patterns. - If `col` is a binary, it is converted to a string in one of the formats: - 'base64': a base 64 string. - 'hex': a string in the hexadecimal format. - 'utf-8': the input binary is decoded to UTF-8 string. + Example 2: Getting an element at a position outside the array boundaries - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.get(df.data, 3)).show() + +------------+ + |get(data, 3)| + +------------+ + | NULL| + +------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - The value to convert to a string. - A column that evaluates to a numeric, date, timestamp, time, or binary. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert char values. - A column that evaluates to a string. Must be a constant when ``col`` is numeric - or binary. + Example 3: Getting an element at a position specified by another column - Examples - -------- - >>> df = spark.createDataFrame([(78.12,)], ["e"]) - >>> df.select(to_varchar(df.e, lit("$99.99")).alias('r')).collect() - [Row(r='$78.12')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) + >>> df.select(sf.get(df.data, df.index)).show() + +----------------+ + |get(data, index)| + +----------------+ + | c| + +----------------+ + + + Example 4: Getting an element at a position calculated from another column + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) + >>> df.select(sf.get(df.data, df.index - 1)).show() + +----------------------+ + |get(data, (index - 1))| + +----------------------+ + | b| + +----------------------+ + + Example 5: Getting an element at a negative position + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"], )], ['data']) + >>> df.select(sf.get(df.data, -1)).show() + +-------------+ + |get(data, -1)| + +-------------+ + | NULL| + +-------------+ """ - return _invoke_function_over_columns("to_varchar", col, format) + index = _enum_to_value(index) + index = lit(index) if isinstance(index, int) else index + + return _invoke_function_over_columns("get", col, index) @_try_remote_functions -def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: +def array_prepend(col: "ColumnOrName", value: Any) -> Column: """ - Convert string 'col' to a number based on the string format 'format'. - Throws an exception if the conversion fails. The format can consist of the following - characters, case insensitive: - '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the - format string matches a sequence of digits in the input string. If the 0/9 - sequence starts with 0 and is before the decimal point, it can only match a digit - sequence of the same size. Otherwise, if the sequence starts with 9 or is after - the decimal point, it can match a digit sequence that has the same or smaller size. - '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). - ',' or 'G': Specifies the position of the grouping (thousands) separator (,). - There must be a 0 or 9 to the left and right of each grouping separator. - 'col' must match the grouping separator relevant for the size of the number. - '$': Specifies the location of the $ currency sign. This character may only be - specified once. - 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed - once at the beginning or end of the format string). Note that 'S' allows '-' - but 'MI' does not. - 'PR': Only allowed at the end of the format string; specifies that 'col' indicates a - negative number with wrapping angled brackets. + Array function: Returns an array containing the given element as + the first element and the rest of the elements from the original array. .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert number values. - A column that evaluates to a string. Must be a constant. + name of column containing array. + A column that evaluates to an array. + value : + a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. + + Returns + ------- + :class:`~pyspark.sql.Column` + an array with the given value prepended. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.try_to_number` + :meth:`pyspark.sql.functions.array_append` + :meth:`pyspark.sql.functions.array_insert` Examples -------- - >>> df = spark.createDataFrame([("$78.12",)], ["e"]) - >>> df.select(to_number(df.e, lit("$99.99")).alias('r')).collect() - [Row(r=Decimal('78.12'))] - """ - return _invoke_function_over_columns("to_number", col, format) + Example 1: Prepending a column value to an array column + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) + >>> df.select(sf.array_prepend(df.c1, df.c2)).show() + +---------------------+ + |array_prepend(c1, c2)| + +---------------------+ + | [c, b, a, c]| + +---------------------+ -@_try_remote_functions -def replace( - src: "ColumnOrName", search: "ColumnOrName", replace: Optional["ColumnOrName"] = None -) -> Column: - """ - Replaces all occurrences of `search` with `replace`. + Example 2: Prepending a numeric value to an array column - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_prepend(df.data, 4)).show() + +----------------------+ + |array_prepend(data, 4)| + +----------------------+ + | [4, 1, 2, 3]| + +----------------------+ - Parameters - ---------- - src : :class:`~pyspark.sql.Column` or str - A column of string to be replaced. - A column that evaluates to a string. - search : :class:`~pyspark.sql.Column` or str - A column of string, If `search` is not found in `str`, `str` is returned unchanged. - A column that evaluates to a string. - replace : :class:`~pyspark.sql.Column` or str, optional - A column of string, If `replace` is not specified or is an empty string, - nothing replaces the string that is removed from `str`. - A column that evaluates to a string. + Example 3: Prepending a null value to an array column - Examples - -------- - >>> df = spark.createDataFrame([("ABCabc", "abc", "DEF",)], ["a", "b", "c"]) - >>> df.select(replace(df.a, df.b, df.c).alias('r')).collect() - [Row(r='ABCDEF')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_prepend(df.data, None)).show() + +-------------------------+ + |array_prepend(data, NULL)| + +-------------------------+ + | [NULL, 1, 2, 3]| + +-------------------------+ - >>> df.select(replace(df.a, df.b).alias('r')).collect() - [Row(r='ABC')] + Example 4: Prepending a value to a NULL array column + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([(None,)], schema=schema) + >>> df.select(sf.array_prepend(df.data, 4)).show() + +----------------------+ + |array_prepend(data, 4)| + +----------------------+ + | NULL| + +----------------------+ + + Example 5: Prepending a value to an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_prepend(df.data, 1)).show() + +----------------------+ + |array_prepend(data, 1)| + +----------------------+ + | [1]| + +----------------------+ """ - if replace is not None: - return _invoke_function_over_columns("replace", src, search, replace) - else: - return _invoke_function_over_columns("replace", src, search) + return _invoke_function_over_columns("array_prepend", col, lit(value)) @_try_remote_functions -def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnOrName") -> Column: +def array_remove(col: "ColumnOrName", element: Any) -> Column: """ - Splits `str` by delimiter and return requested part of the split (1-based). - If any input is null, returns null. if `partNum` is out of range of split parts, - returns empty string. If `partNum` is 0, throws an error. If `partNum` is negative, - the parts are counted backward from the end of the string. - If the `delimiter` is an empty string, the `str` is not split. + Array function: Remove all elements that equal to element from the given array. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - src : :class:`~pyspark.sql.Column` or column name - A column of string to be split. - A column that evaluates to a string. - delimiter : :class:`~pyspark.sql.Column` or column name - A column of string, the delimiter used for split. - A column that evaluates to a string. - partNum : :class:`~pyspark.sql.Column` or column name - The requested part of the split (1-based). - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + name of column containing array. + A column that evaluates to an array. + element : + element or a :class:`~pyspark.sql.Column` expression to be removed from the array. + A column of the same type as the array elements. + + .. versionchanged:: 4.0.0 + `element` now also accepts a Column type. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that is an array excluding the given value from the input column. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.sentences` - :meth:`pyspark.sql.functions.split` + :meth:`pyspark.sql.functions.array_compact` Examples -------- + Example 1: Removing a specific value from a simple array + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("11.12.13", ".", 3,)], ["a", "b", "c"]) - >>> df.select("*", sf.split_part("a", "b", "c")).show() - +--------+---+---+-------------------+ - | a| b| c|split_part(a, b, c)| - +--------+---+---+-------------------+ - |11.12.13| .| 3| 13| - +--------+---+---+-------------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],)], ['data']) + >>> df.select(sf.array_remove(df.data, 1)).show() + +---------------------+ + |array_remove(data, 1)| + +---------------------+ + | [2, 3]| + +---------------------+ - >>> df.select("*", sf.split_part(df.a, df.b, sf.lit(-2))).show() - +--------+---+---+--------------------+ - | a| b| c|split_part(a, b, -2)| - +--------+---+---+--------------------+ - |11.12.13| .| 3| 12| - +--------+---+---+--------------------+ - """ - return _invoke_function_over_columns("split_part", src, delimiter, partNum) + Example 2: Removing a specific value from multiple arrays + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],), ([4, 5, 5, 4],)], ['data']) + >>> df.select(sf.array_remove(df.data, 5)).show() + +---------------------+ + |array_remove(data, 5)| + +---------------------+ + | [1, 2, 3, 1, 1]| + | [4, 4]| + +---------------------+ -@_try_remote_functions -def substr( - str: "ColumnOrName", pos: "ColumnOrName", len: Optional["ColumnOrName"] = None -) -> Column: - """ - Returns the substring of `str` that starts at `pos` and is of length `len`, - or the slice of byte array that starts at `pos` and is of length `len`. + Example 3: Removing a value that does not exist in the array - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_remove(df.data, 4)).show() + +---------------------+ + |array_remove(data, 4)| + +---------------------+ + | [1, 2, 3]| + +---------------------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of string. - A column that evaluates to a string or binary. - pos : :class:`~pyspark.sql.Column` or column name - The starting position of the substring. - A column that evaluates to an integer. - len : :class:`~pyspark.sql.Column` or column name, optional - The length of the substring. - A column that evaluates to an integer. + Example 4: Removing a value from an array with all identical values - Returns - ------- - :class:`~pyspark.sql.Column` - substring of given value. - Returns a column of the same type as the input. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) + >>> df.select(sf.array_remove(df.data, 1)).show() + +---------------------+ + |array_remove(data, 1)| + +---------------------+ + | []| + +---------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.Column.substr` - :meth:`pyspark.sql.functions.locate` + Example 5: Removing a value from an empty array - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Spark SQL", 5, 1,)], ["a", "b", "c"]) - >>> df.select("*", sf.substr("a", "b", "c")).show() - +---------+---+---+---------------+ - | a| b| c|substr(a, b, c)| - +---------+---+---+---------------+ - |Spark SQL| 5| 1| k| - +---------+---+---+---------------+ + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema) + >>> df.select(sf.array_remove(df.data, 1)).show() + +---------------------+ + |array_remove(data, 1)| + +---------------------+ + | []| + +---------------------+ - >>> df.select("*", sf.substr(df.a, df.b)).show() - +---------+---+---+------------------------+ - | a| b| c|substr(a, b, 2147483647)| - +---------+---+---+------------------------+ - |Spark SQL| 5| 1| k SQL| - +---------+---+---+------------------------+ + Example 6: Removing a column's value from a simple array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3, 1, 1], 1)], ['data', 'col']) + >>> df.select(sf.array_remove(df.data, df.col)).show() + +-----------------------+ + |array_remove(data, col)| + +-----------------------+ + | [2, 3]| + +-----------------------+ """ - if len is not None: - return _invoke_function_over_columns("substr", str, pos, len) - else: - return _invoke_function_over_columns("substr", str, pos) + return _invoke_function_over_columns("array_remove", col, lit(element)) @_try_remote_functions -def try_parse_url( - url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None -) -> Column: +def array_distinct(col: "ColumnOrName") -> Column: """ - This is a special version of `parse_url` that performs the same operation, but returns a - NULL value instead of raising an error if the parsing cannot be performed. + Array function: removes duplicate values from the array. - .. versionadded:: 4.0.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - url : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL. - A column that evaluates to a string. - partToExtract : :class:`~pyspark.sql.Column` or str - A column of strings, each representing the part to extract from the URL. - A column that evaluates to a string. - key : :class:`~pyspark.sql.Column` or str, optional - A column of strings, each representing the key of a query parameter in the URL. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the value of the extracted part from the URL. - Returns a column that evaluates to a string. + A new column that is an array of unique values from the input column. + Returns a column that evaluates to an array. + + See Also + -------- + :meth:`pyspark.sql.functions.array_except` + :meth:`pyspark.sql.functions.array_intersect` + :meth:`pyspark.sql.functions.array_union` Examples -------- - Example 1: Extracting the query part from a URL + Example 1: Removing duplicate values from a simple array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | query=1| - +------------------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 2],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1, 2, 3]| + +--------------------+ - Example 2: Extracting the value of a specific query parameter from a URL + Example 2: Removing duplicate values from multiple arrays >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], - ... ["url", "part", "key"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() - +-----------------------------+ - |try_parse_url(url, part, key)| - +-----------------------------+ - | 1| - +-----------------------------+ - - Example 3: Extracting the protocol part from a URL - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | https| - +------------------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1, 2, 3]| + | [4, 5]| + +--------------------+ - Example 4: Extracting the host part from a URL + Example 3: Removing duplicate values from an array with all identical values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "HOST")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | spark.apache.org| - +------------------------+ + >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1]| + +--------------------+ - Example 5: Extracting the path part from a URL + Example 4: Removing duplicate values from an array with no duplicate values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PATH")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | /path| - +------------------------+ + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1, 2, 3]| + +--------------------+ - Example 6: Invalid URL + Example 5: Removing duplicate values from an empty array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("inva lid://spark.apache.org/path?query=1", "QUERY", "query")], - ... ["url", "part", "key"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() - +-----------------------------+ - |try_parse_url(url, part, key)| - +-----------------------------+ - | NULL| - +-----------------------------+ + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | []| + +--------------------+ """ - if key is not None: - return _invoke_function_over_columns("try_parse_url", url, partToExtract, key) - else: - return _invoke_function_over_columns("try_parse_url", url, partToExtract) + return _invoke_function_over_columns("array_distinct", col) @_try_remote_functions -def parse_url( - url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None -) -> Column: +def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: Any) -> Column: """ - URL function: Extracts a specified part from a URL. If a key is provided, - it returns the associated query parameter value. + Array function: Inserts an item into a given array at a specified array index. + Array indices start at 1, or start from the end if index is negative. + Index above array size appends the array, or prepends the array if index is negative, + with 'null' elements. - .. versionadded:: 3.5.0 + .. versionadded:: 3.4.0 Parameters ---------- - url : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL. - A column that evaluates to a string. - partToExtract : :class:`~pyspark.sql.Column` or str - A column of strings, each representing the part to extract from the URL. - A column that evaluates to a string. - key : :class:`~pyspark.sql.Column` or str, optional - A column of strings, each representing the key of a query parameter in the URL. - A column that evaluates to a string. + arr : :class:`~pyspark.sql.Column` or str + name of column containing an array. + A column that evaluates to an array. + pos : :class:`~pyspark.sql.Column` or str or int + name of integral type column indicating position of insertion + (starting at index 1, negative position is a start from the back of the array). + A column that evaluates to an integer. + value : + a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the value of the extracted part from the URL. - Returns a column that evaluates to a string. + an array of values, including the new specified value + Returns a column that evaluates to an array. + + Notes + ----- + Supports Spark Connect. + + See Also + -------- + :meth:`pyspark.sql.functions.array_append` + :meth:`pyspark.sql.functions.array_prepend` Examples -------- - Example 1: Extracting the query part from a URL + Example 1: Inserting a value at a specific position >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | query=1| - +--------------------+ + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, 2, 'd')).show() + +------------------------+ + |array_insert(data, 2, d)| + +------------------------+ + | [a, d, b, c]| + +------------------------+ - Example 2: Extracting the value of a specific query parameter from a URL + Example 2: Inserting a value at a negative position >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], - ... ["url", "part", "key"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part, df.key)).show() + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, -2, 'd')).show() +-------------------------+ - |parse_url(url, part, key)| + |array_insert(data, -2, d)| +-------------------------+ - | 1| + | [a, b, d, c]| +-------------------------+ - Example 3: Extracting the protocol part from a URL + Example 3: Inserting a value at a position greater than the array size >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | https| - +--------------------+ + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, 5, 'e')).show() + +------------------------+ + |array_insert(data, 5, e)| + +------------------------+ + | [a, b, c, NULL, e]| + +------------------------+ - Example 4: Extracting the host part from a URL + Example 4: Inserting a NULL value >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "HOST")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | spark.apache.org| - +--------------------+ + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, 2, sf.lit(None))).show() + +---------------------------+ + |array_insert(data, 2, NULL)| + +---------------------------+ + | [a, NULL, b, c]| + +---------------------------+ - Example 5: Extracting the path part from a URL + Example 5: Inserting a value into a NULL array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PATH")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | /path| - +--------------------+ + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) + >>> df = spark.createDataFrame([(None,)], schema=schema) + >>> df.select(sf.array_insert(df.data, 1, 5)).show() + +------------------------+ + |array_insert(data, 1, 5)| + +------------------------+ + | NULL| + +------------------------+ """ - if key is not None: - return _invoke_function_over_columns("parse_url", url, partToExtract, key) - else: - return _invoke_function_over_columns("parse_url", url, partToExtract) + pos = _enum_to_value(pos) + pos = lit(pos) if isinstance(pos, int) else pos + + return _invoke_function_over_columns("array_insert", arr, pos, lit(value)) @_try_remote_functions -def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: +def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Formats the arguments in printf-style and returns the result as a string column. + Array function: returns a new array containing the intersection of elements in col1 and col2, + without duplicates. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - format : :class:`~pyspark.sql.Column` or str - string that can contain embedded format tags and used as result column's value. - A column that evaluates to a string. - cols : :class:`~pyspark.sql.Column` or str - column names or :class:`~pyspark.sql.Column`\\s to be used in formatting - Each a column of any type. + col1 : :class:`~pyspark.sql.Column` or str + Name of column containing the first array. + A column that evaluates to an array. + col2 : :class:`~pyspark.sql.Column` or str + Name of column containing the second array. + A column that evaluates to an array. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new array containing the intersection of elements in col1 and col2. + Returns a column that evaluates to an array. + + Notes + ----- + This function does not preserve the order of the elements in the input arrays. See Also -------- - :meth:`pyspark.sql.functions.format_string` + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.array_except` + :meth:`pyspark.sql.functions.array_union` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("aa%d%s", 123, "cc",)], ["a", "b", "c"] - ... ).select(sf.printf("a", "b", "c")).show() - +---------------+ - |printf(a, b, c)| - +---------------+ - | aa123cc| - +---------------+ - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + Example 1: Basic usage - sc = _get_active_spark_context() - return _invoke_function("printf", _to_java_column(format), _to_seq(sc, cols, _to_java_column)) + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) + >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() + +-----------------------------------------+ + |sort_array(array_intersect(c1, c2), true)| + +-----------------------------------------+ + | [a, c]| + +-----------------------------------------+ + + Example 2: Intersection with no common elements + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) + >>> df.select(sf.array_intersect(df.c1, df.c2)).show() + +-----------------------+ + |array_intersect(c1, c2)| + +-----------------------+ + | []| + +-----------------------+ + + Example 3: Intersection with all common elements + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) + >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() + +-----------------------------------------+ + |sort_array(array_intersect(c1, c2), true)| + +-----------------------------------------+ + | [a, b, c]| + +-----------------------------------------+ + + Example 4: Intersection with null values + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) + >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() + +-----------------------------------------+ + |sort_array(array_intersect(c1, c2), true)| + +-----------------------------------------+ + | [NULL, a]| + +-----------------------------------------+ + + Example 5: Intersection with empty arrays + + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> data = [Row(c1=[], c2=["a", "b", "c"])] + >>> schema = StructType([ + ... StructField("c1", ArrayType(StringType()), True), + ... StructField("c2", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.array_intersect(df.c1, df.c2)).show() + +-----------------------+ + |array_intersect(c1, c2)| + +-----------------------+ + | []| + +-----------------------+ + """ + return _invoke_function_over_columns("array_intersect", col1, col2) @_try_remote_functions -def url_decode(str: "ColumnOrName") -> Column: +def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - URL function: Decodes a URL-encoded string in 'application/x-www-form-urlencoded' - format to its original format. + Array function: returns a new array containing the union of elements in col1 and col2, + without duplicates. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL-encoded string. - A column that evaluates to a string. + col1 : :class:`~pyspark.sql.Column` or str + Name of column containing the first array. + A column that evaluates to an array. + col2 : :class:`~pyspark.sql.Column` or str + Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the decoded string. - Returns a column that evaluates to a string. + A new array containing the union of elements in col1 and col2. + Returns a column that evaluates to an array. + + Notes + ----- + This function does not preserve the order of the elements in the input arrays. + + See Also + -------- + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.array_except` + :meth:`pyspark.sql.functions.array_intersect` Examples -------- - Example 1: Decoding a URL-encoded string + Example 1: Basic usage - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show(truncate=False) - +------------------------+ - |url_decode(url) | - +------------------------+ - |https://spark.apache.org| - +------------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c, d, f]| + +-------------------------------------+ - Example 2: Decoding a URL-encoded string with spaces + Example 2: Union with no common elements - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Hello%20World%21",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | Hello World!| - +---------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c, d, e, f]| + +-------------------------------------+ - Example 3: Decoding a URL-encoded string with special characters + Example 3: Union with all common elements - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("A%2BB%3D%3D",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | A+B==| - +---------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c]| + +-------------------------------------+ - Example 4: Decoding a URL-encoded string with non-ASCII characters + Example 4: Union with null values - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("%E4%BD%A0%E5%A5%BD",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | 你好| - +---------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [NULL, a, b, c]| + +-------------------------------------+ - Example 5: Decoding a URL-encoded string with hexadecimal values + Example 5: Union with empty arrays - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("%7E%21%40%23%24%25%5E%26%2A%28%29%5F%2B",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | ~!@#$%^&*()_+| - +---------------+ + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> data = [Row(c1=[], c2=["a", "b", "c"])] + >>> schema = StructType([ + ... StructField("c1", ArrayType(StringType()), True), + ... StructField("c2", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c]| + +-------------------------------------+ """ - return _invoke_function_over_columns("url_decode", str) + return _invoke_function_over_columns("array_union", col1, col2) @_try_remote_functions -def try_url_decode(str: "ColumnOrName") -> Column: +def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - This is a special version of `url_decode` that performs the same operation, but returns a - NULL value instead of raising an error if the decoding cannot be performed. + Array function: returns a new array containing the elements present in col1 but not in col2, + without duplicates. - .. versionadded:: 4.0.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL-encoded string. - A column that evaluates to a string. + col1 : :class:`~pyspark.sql.Column` or str + Name of column containing the first array. + A column that evaluates to an array. + col2 : :class:`~pyspark.sql.Column` or str + Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the decoded string. - Returns a column that evaluates to a string. + A new array containing the elements present in col1 but not in col2. + Returns a column that evaluates to an array. - Examples - -------- - Example 1: Decoding a URL-encoded string + Notes + ----- + This function does not preserve the order of the elements in the input arrays. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) - >>> df.select(sf.try_url_decode(df.url)).show(truncate=False) - +------------------------+ - |try_url_decode(url) | - +------------------------+ - |https://spark.apache.org| - +------------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.array_intersect` + :meth:`pyspark.sql.functions.array_union` - Example 2: Return NULL if the decoding cannot be performed. + Examples + -------- + Example 1: Basic usage - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https%3A%2F%2spark.apache.org",)], ["url"]) - >>> df.select(sf.try_url_decode(df.url)).show() - +-------------------+ - |try_url_decode(url)| - +-------------------+ - | NULL| - +-------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) + >>> df.select(sf.array_except(df.c1, df.c2)).show() + +--------------------+ + |array_except(c1, c2)| + +--------------------+ + | [b]| + +--------------------+ + + Example 2: Except with no common elements + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) + >>> df.select(sf.sort_array(sf.array_except(df.c1, df.c2))).show() + +--------------------------------------+ + |sort_array(array_except(c1, c2), true)| + +--------------------------------------+ + | [a, b, c]| + +--------------------------------------+ + + Example 3: Except with all common elements + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) + >>> df.select(sf.array_except(df.c1, df.c2)).show() + +--------------------+ + |array_except(c1, c2)| + +--------------------+ + | []| + +--------------------+ + + Example 4: Except with null values + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) + >>> df.select(sf.array_except(df.c1, df.c2)).show() + +--------------------+ + |array_except(c1, c2)| + +--------------------+ + | [b]| + +--------------------+ + + Example 5: Except with empty arrays + + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> data = [Row(c1=[], c2=["a", "b", "c"])] + >>> schema = StructType([ + ... StructField("c1", ArrayType(StringType()), True), + ... StructField("c2", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.array_except(df.c1, df.c2)).show() + +--------------------+ + |array_except(c1, c2)| + +--------------------+ + | []| + +--------------------+ """ - return _invoke_function_over_columns("try_url_decode", str) + return _invoke_function_over_columns("array_except", col1, col2) @_try_remote_functions -def url_encode(str: "ColumnOrName") -> Column: +def array_compact(col: "ColumnOrName") -> Column: """ - URL function: Encodes a string into a URL-encoded string in - 'application/x-www-form-urlencoded' format. + Array function: removes null values from the array. - .. versionadded:: 3.5.0 + .. versionadded:: 3.4.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a string to be URL-encoded. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the URL-encoded string. - Returns a column that evaluates to a string. + A new column that is an array excluding the null values from the input column. + Returns a column that evaluates to an array. + + Notes + ----- + Supports Spark Connect. + + See Also + -------- + :meth:`pyspark.sql.functions.array_remove` Examples -------- - Example 1: Encoding a simple URL + Example 1: Removing null values from a simple array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https://spark.apache.org",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show(truncate=False) - +------------------------------+ - |url_encode(url) | - +------------------------------+ - |https%3A%2F%2Fspark.apache.org| - +------------------------------+ + >>> df = spark.createDataFrame([([1, None, 2, 3],)], ['data']) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | [1, 2, 3]| + +-------------------+ - Example 2: Encoding a URL with spaces + Example 2: Removing null values from multiple arrays >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Hello World!",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show() - +---------------+ - |url_encode(url)| - +---------------+ - | Hello+World%21| - +---------------+ + >>> df = spark.createDataFrame([([1, None, 2, 3],), ([4, 5, None, 4],)], ['data']) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | [1, 2, 3]| + | [4, 5, 4]| + +-------------------+ - Example 3: Encoding a URL with special characters + Example 3: Removing null values from an array with all null values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("A+B==",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show() - +---------------+ - |url_encode(url)| - +---------------+ - | A%2BB%3D%3D| - +---------------+ + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([ + ... StructField("data", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame([([None, None, None],)], schema) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | []| + +-------------------+ - Example 4: Encoding a URL with non-ASCII characters + Example 4: Removing null values from an array with no null values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("你好",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show() - +------------------+ - | url_encode(url)| - +------------------+ - |%E4%BD%A0%E5%A5%BD| - +------------------+ + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | [1, 2, 3]| + +-------------------+ - Example 5: Encoding a URL with hexadecimal values + Example 5: Removing null values from an empty array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("~!@#$%^&*()_+",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show(truncate=False) - +-----------------------------------+ - |url_encode(url) | - +-----------------------------------+ - |%7E%21%40%23%24%25%5E%26*%28%29_%2B| - +-----------------------------------+ + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([ + ... StructField("data", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | []| + +-------------------+ """ - return _invoke_function_over_columns("url_encode", str) + return _invoke_function_over_columns("array_compact", col) @_try_remote_functions -def position( - substr: "ColumnOrName", str: "ColumnOrName", start: Optional["ColumnOrName"] = None -) -> Column: +def array_append(col: "ColumnOrName", value: Any) -> Column: """ - Returns the position of the first occurrence of `substr` in `str` after position `start`. - The given `start` and return value are 1-based. + Array function: returns a new array column by appending `value` to the existing array `col`. - .. versionadded:: 3.5.0 + .. versionadded:: 3.4.0 Parameters ---------- - substr : :class:`~pyspark.sql.Column` or str - A column of string, substring. - A column that evaluates to a string. - str : :class:`~pyspark.sql.Column` or str - A column of string. - A column that evaluates to a string. - start : :class:`~pyspark.sql.Column` or str, optional - The start position. - A column that evaluates to an integer. - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] - ... ).select(sf.position("a", "b", "c")).show() - +-----------------+ - |position(a, b, c)| - +-----------------+ - | 7| - +-----------------+ - - >>> spark.createDataFrame( - ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] - ... ).select(sf.position("a", "b")).show() - +-----------------+ - |position(a, b, 1)| - +-----------------+ - | 4| - +-----------------+ - """ - if start is not None: - return _invoke_function_over_columns("position", substr, str, start) - else: - return _invoke_function_over_columns("position", substr, str) - + col : :class:`~pyspark.sql.Column` or str + The name of the column containing the array. + A column that evaluates to an array. + value : + A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. + A column of the same type as the array elements. -@_try_remote_functions -def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: - """ - Returns a boolean. The value is True if str ends with suffix. - Returns NULL if either input expression is NULL. Otherwise, returns False. - Both str or suffix must be of STRING or BINARY type. + Returns + ------- + :class:`~pyspark.sql.Column` + A new array column with `value` appended to the original array. + Returns a column that evaluates to an array. - .. versionadded:: 3.5.0 + Notes + ----- + Supports Spark Connect. - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - The input value to test. - A column that evaluates to a string or binary. - suffix : :class:`~pyspark.sql.Column` or str - The suffix to test for. - A column that evaluates to a string or binary. + See Also + -------- + :meth:`pyspark.sql.functions.array_insert` + :meth:`pyspark.sql.functions.array_prepend` Examples -------- - >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) - >>> df.select(endswith(df.a, df.b).alias('r')).collect() - [Row(r=False)] + Example 1: Appending a column value to an array column - >>> df = spark.createDataFrame([("414243", "4243",)], ["e", "f"]) - >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) - >>> df.printSchema() - root - |-- e: binary (nullable = true) - |-- f: binary (nullable = true) - >>> df.select(endswith("e", "f"), endswith("f", "e")).show() - +--------------+--------------+ - |endswith(e, f)|endswith(f, e)| - +--------------+--------------+ - | true| false| - +--------------+--------------+ - """ - return _invoke_function_over_columns("endswith", str, suffix) + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) + >>> df.select(sf.array_append(df.c1, df.c2)).show() + +--------------------+ + |array_append(c1, c2)| + +--------------------+ + | [b, a, c, c]| + +--------------------+ + Example 2: Appending a numeric value to an array column -@_try_remote_functions -def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: - """ - Returns a boolean. The value is True if str starts with prefix. - Returns NULL if either input expression is NULL. Otherwise, returns False. - Both str or prefix must be of STRING or BINARY type. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_append(df.data, 4)).show() + +---------------------+ + |array_append(data, 4)| + +---------------------+ + | [1, 2, 3, 4]| + +---------------------+ - .. versionadded:: 3.5.0 + Example 3: Appending a null value to an array column - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - The input value to test. - A column that evaluates to a string or binary. - prefix : :class:`~pyspark.sql.Column` or str - The prefix to test for. - A column that evaluates to a string or binary. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_append(df.data, None)).show() + +------------------------+ + |array_append(data, NULL)| + +------------------------+ + | [1, 2, 3, NULL]| + +------------------------+ - Examples - -------- - >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) - >>> df.select(startswith(df.a, df.b).alias('r')).collect() - [Row(r=True)] + Example 4: Appending a value to a NULL array column - >>> df = spark.createDataFrame([("414243", "4142",)], ["e", "f"]) - >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) - >>> df.printSchema() - root - |-- e: binary (nullable = true) - |-- f: binary (nullable = true) - >>> df.select(startswith("e", "f"), startswith("f", "e")).show() - +----------------+----------------+ - |startswith(e, f)|startswith(f, e)| - +----------------+----------------+ - | true| false| - +----------------+----------------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([(None,)], schema=schema) + >>> df.select(sf.array_append(df.data, 4)).show() + +---------------------+ + |array_append(data, 4)| + +---------------------+ + | NULL| + +---------------------+ + + Example 5: Appending a value to an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_append(df.data, 1)).show() + +---------------------+ + |array_append(data, 1)| + +---------------------+ + | [1]| + +---------------------+ """ - return _invoke_function_over_columns("startswith", str, prefix) + return _invoke_function_over_columns("array_append", col, lit(value)) @_try_remote_functions -def char(col: "ColumnOrName") -> Column: +def array_min(col: "ColumnOrName") -> Column: """ - Returns the ASCII character having the binary equivalent to `col`. If col is larger than 256 the - result is equivalent to char(col % 256) + Array function: returns the minimum value of the array. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a long. + The name of the column or an expression that represents the array. + A column that evaluates to an array. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains the minimum value of each array. + Returns a column of the element type of the input array. + + See Also + -------- + :meth:`pyspark.sql.functions.array_max` + :meth:`pyspark.sql.functions.array_sort` + :meth:`pyspark.sql.functions.sort_array` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.char(sf.lit(65))).show() - +--------+ - |char(65)| - +--------+ - | A| - +--------+ - """ - return _invoke_function_over_columns("char", col) + Example 1: Basic usage with integer array + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | 1| + | -1| + +---------------+ -@_try_remote_functions -def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: - """ - Remove the leading and trailing `trim` characters from `str`. + Example 2: Usage with string array - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | apple| + +---------------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or str, optional - The trim string characters to trim, the default value is a single space. - A column that evaluates to a string. + Example 3: Usage with mixed type array - Examples - -------- - >>> df = spark.createDataFrame([("SSparkSQLS", "SL", )], ['a', 'b']) - >>> df.select(btrim(df.a, df.b).alias('r')).collect() - [Row(r='parkSQ')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | 1| + +---------------+ - >>> df = spark.createDataFrame([(" SparkSQL ",)], ['a']) - >>> df.select(btrim(df.a).alias('r')).collect() - [Row(r='SparkSQL')] + Example 4: Usage with array of arrays + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | [2, 1]| + +---------------+ + + Example 5: Usage with empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | NULL| + +---------------+ """ - if trim is not None: - return _invoke_function_over_columns("btrim", str, trim) - else: - return _invoke_function_over_columns("btrim", str) + return _invoke_function_over_columns("array_min", col) @_try_remote_functions -def char_length(str: "ColumnOrName") -> Column: +def array_max(col: "ColumnOrName") -> Column: """ - Returns the character length of string data or number of bytes of binary data. - The length of string data includes the trailing spaces. - The length of binary data includes binary zeros. + Array function: returns the maximum value of the array. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string or binary. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the array. + A column that evaluates to an array. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains the maximum value of each array. + Returns a column of the element type of the input array. See Also -------- - :meth:`pyspark.sql.functions.character_length` - :meth:`pyspark.sql.functions.length` + :meth:`pyspark.sql.functions.array_min` + :meth:`pyspark.sql.functions.array_sort` + :meth:`pyspark.sql.functions.sort_array` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.char_length(sf.lit("SparkSQL"))).show() - +---------------------+ - |char_length(SparkSQL)| - +---------------------+ - | 8| - +---------------------+ + Example 1: Basic usage with integer array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | 3| + | 10| + +---------------+ + + Example 2: Usage with string array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | cherry| + +---------------+ + + Example 3: Usage with mixed type array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | cherry| + +---------------+ + + Example 4: Usage with array of arrays + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | [3, 4]| + +---------------+ + + Example 5: Usage with empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | NULL| + +---------------+ """ - return _invoke_function_over_columns("char_length", str) + return _invoke_function_over_columns("array_max", col) @_try_remote_functions -def character_length(str: "ColumnOrName") -> Column: +def array_size(col: "ColumnOrName") -> Column: """ - Returns the character length of string data or number of bytes of binary data. - The length of string data includes the trailing spaces. - The length of binary data includes binary zeros. + Array function: returns the total number of elements in the array. + The function returns null for null input. .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string or binary. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the array. + A column that evaluates to an array. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains the size of each array. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.char_length` - :meth:`pyspark.sql.functions.length` + :meth:`pyspark.sql.functions.cardinality` + :meth:`pyspark.sql.functions.size` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.character_length(sf.lit("SparkSQL"))).show() - +--------------------------+ - |character_length(SparkSQL)| - +--------------------------+ - | 8| - +--------------------------+ - """ - return _invoke_function_over_columns("character_length", str) + Example 1: Basic usage with integer array + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([2, 1, 3],), (None,)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 3| + | NULL| + +----------------+ -@_try_remote_functions -def chr(n: "ColumnOrName") -> Column: - """ - Returns the ASCII character having the binary equivalent to `n`. - If n is larger than 256 the result is equivalent to chr(n % 256). + Example 2: Usage with string array - .. versionadded:: 4.1.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 3| + +----------------+ - Parameters - ---------- - n : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a long. + Example 3: Usage with mixed type array - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(60, 70).select("*", sf.chr("id")).show() - +---+-------+ - | id|chr(id)| - +---+-------+ - | 60| <| - | 61| =| - | 62| >| - | 63| ?| - | 64| @| - | 65| A| - | 66| B| - | 67| C| - | 68| D| - | 69| E| - +---+-------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 3| + +----------------+ + + Example 4: Usage with array of arrays + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 2| + +----------------+ + + Example 5: Usage with empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 0| + +----------------+ """ - return _invoke_function_over_columns("chr", n) + return _invoke_function_over_columns("array_size", col) @_try_remote_functions -def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: +def sort_array(col: "ColumnOrName", asc: bool = True) -> Column: """ - This is a special version of `to_binary` that performs the same operation, but returns a NULL - value instead of raising an error if the conversion cannot be performed. + Array function: Sorts the input array in ascending or descending order according + to the natural ordering of the array elements. Null elements will be placed at the beginning + of the returned array in ascending order or at the end of the returned array in descending + order. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert binary values. - A column that evaluates to a string. Must be a constant. + Name of the column or expression. + A column that evaluates to an array. + asc : bool, optional + Whether to sort in ascending or descending order. If `asc` is True (default), + then the sorting is in ascending order. If False, then in descending order. + A column that evaluates to a boolean. Must be a constant. - See Also - -------- - :meth:`pyspark.sql.functions.to_binary` + Returns + ------- + :class:`~pyspark.sql.Column` + Sorted array. + Returns a column that evaluates to an array. Examples -------- - Example 1: Convert string to a binary with encoding specified + Example 1: Sorting an array in ascending order >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("abc",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() - [Row(r=b'abc')] + >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | [NULL, 1, 2, 3]| + +----------------------+ - Example 2: Convert string to a timestamp without encoding specified + Example 2: Sorting an array in descending order >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("414243",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() - [Row(r=b'ABC')] + >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) + >>> df.select(sf.sort_array(df.data, asc=False)).show() + +-----------------------+ + |sort_array(data, false)| + +-----------------------+ + | [3, 2, 1, NULL]| + +-----------------------+ - Example 3: Converion failure results in NULL when ANSI mode is on + Example 3: Sorting an array with a single element >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.range(1) - ... df.select(sf.try_to_binary(sf.lit("malformed"), sf.lit("hex"))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-----------------------------+ - |try_to_binary(malformed, hex)| - +-----------------------------+ - | NULL| - +-----------------------------+ + >>> df = spark.createDataFrame([([1],)], ['data']) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | [1]| + +----------------------+ + + Example 4: Sorting an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | []| + +----------------------+ + + Example 5: Sorting an array with null values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) + >>> df = spark.createDataFrame([([None, None, None],)], schema=schema) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | [NULL, NULL, NULL]| + +----------------------+ """ - if format is not None: - return _invoke_function_over_columns("try_to_binary", col, format) - else: - return _invoke_function_over_columns("try_to_binary", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("sort_array", _to_java_column(col), _enum_to_value(asc)) @_try_remote_functions -def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: +def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> Column: """ - Convert string 'col' to a number based on the string format `format`. Returns NULL if the - string 'col' does not match the expected format. The format follows the same semantics as the - to_number function. + Array function: Generates a random permutation of the given array. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert number values. - A column that evaluates to a string. Must be a constant. + The name of the column or expression to be shuffled. + A column that evaluates to an array. + seed : :class:`~pyspark.sql.Column` or int, optional + Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. - See Also - -------- - :meth:`pyspark.sql.functions.to_number` + .. versionadded:: 4.0.0 + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains an array of elements in random order. + Returns a column that evaluates to an array. + + Notes + ----- + The `shuffle` function is non-deterministic, meaning the order of the output array + can be different for each execution. Examples -------- - Example 1: Convert a string to a number with a format specified + Example 1: Shuffling a simple array >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("$78.12",)], ["e"]) - >>> df.select(sf.try_to_number(df.e, sf.lit("$99.99")).alias('r')).show() - +-----+ - | r| - +-----+ - |78.12| - +-----+ + >>> df = spark.sql("SELECT ARRAY(1, 20, 3, 5) AS data") + >>> df.select("*", sf.shuffle(df.data, sf.lit(123))).show() # doctest: +SKIP + +-------------+-------------+ + | data|shuffle(data)| + +-------------+-------------+ + |[1, 20, 3, 5]|[5, 1, 20, 3]| + +-------------+-------------+ - Example 2: Converion failure results in NULL when ANSI mode is on + Example 2: Shuffling an array with null values >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.range(1) - ... df.select(sf.try_to_number(sf.lit("77"), sf.lit("$99.99")).alias('r')).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +----+ - | r| - +----+ - |NULL| - +----+ + >>> df = spark.sql("SELECT ARRAY(1, 20, NULL, 5) AS data") + >>> df.select("*", sf.shuffle(sf.col("data"), 234)).show() # doctest: +SKIP + +----------------+----------------+ + | data| shuffle(data)| + +----------------+----------------+ + |[1, 20, NULL, 5]|[NULL, 5, 20, 1]| + +----------------+----------------+ + + Example 3: Shuffling an array with duplicate values + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") + >>> df.select("*", sf.shuffle("data", 345)).show() # doctest: +SKIP + +------------------+------------------+ + | data| shuffle(data)| + +------------------+------------------+ + |[1, 2, 2, 3, 3, 3]|[2, 3, 3, 1, 2, 3]| + +------------------+------------------+ + + Example 4: Shuffling an array with random seed + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") + >>> df.select("*", sf.shuffle("data")).show() # doctest: +SKIP + +------------------+------------------+ + | data| shuffle(data)| + +------------------+------------------+ + |[1, 2, 2, 3, 3, 3]|[3, 3, 2, 3, 2, 1]| + +------------------+------------------+ """ - return _invoke_function_over_columns("try_to_number", col, format) + if seed is not None: + return _invoke_function_over_columns("shuffle", col, lit(seed)) + else: + return _invoke_function_over_columns("shuffle", col) @_try_remote_functions -def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def flatten(col: "ColumnOrName") -> Column: """ - Returns a boolean. The value is True if right is found inside left. - Returns NULL if either input expression is NULL. Otherwise, returns False. - Both left or right must be of STRING or BINARY type. + Array function: creates a single array from an array of arrays. + If a structure of nested arrays is deeper than two levels, + only one level of nesting is removed. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - left : :class:`~pyspark.sql.Column` or str - The input to check; may be NULL. - A column that evaluates to a string or binary. - right : :class:`~pyspark.sql.Column` or str - The value to find; may be NULL. - A column that evaluates to a string or binary. + col : :class:`~pyspark.sql.Column` or str + The name of the column or expression to be flattened. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains the flattened array. Examples -------- - >>> df = spark.createDataFrame([("Spark SQL", "Spark")], ['a', 'b']) - >>> df.select(contains(df.a, df.b).alias('r')).collect() - [Row(r=True)] + Example 1: Flattening a simple nested array - >>> df = spark.createDataFrame([("414243", "4243",)], ["c", "d"]) - >>> df = df.select(to_binary("c").alias("c"), to_binary("d").alias("d")) - >>> df.printSchema() - root - |-- c: binary (nullable = true) - |-- d: binary (nullable = true) - >>> df.select(contains("c", "d"), contains("d", "c")).show() - +--------------+--------------+ - |contains(c, d)|contains(d, c)| - +--------------+--------------+ - | true| false| - +--------------+--------------+ - """ - return _invoke_function_over_columns("contains", left, right) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([[1, 2, 3], [4, 5], [6]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show() + +------------------+ + | flatten(data)| + +------------------+ + |[1, 2, 3, 4, 5, 6]| + +------------------+ + Example 2: Flattening an array with null values -@_try_remote_functions -def elt(*inputs: "ColumnOrName") -> Column: - """ - Returns the `n`-th input, e.g., returns `input2` when `n` is 2. - The function returns NULL if the index exceeds the length of the array - and `spark.sql.ansi.enabled` is set to false. If `spark.sql.ansi.enabled` is set to true, - it throws ArrayIndexOutOfBoundsException for invalid indices. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([None, [4, 5]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show() + +-------------+ + |flatten(data)| + +-------------+ + | NULL| + +-------------+ - .. versionadded:: 3.5.0 + Example 3: Flattening an array with more than two levels of nesting - Parameters - ---------- - inputs : :class:`~pyspark.sql.Column` or str - Input columns or strings. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([[[1, 2], [3, 4]], [[5, 6], [7, 8]]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show(truncate=False) + +--------------------------------+ + |flatten(data) | + +--------------------------------+ + |[[1, 2], [3, 4], [5, 6], [7, 8]]| + +--------------------------------+ - Examples - -------- - >>> df = spark.createDataFrame([(1, "scala", "java")], ['a', 'b', 'c']) - >>> df.select(elt(df.a, df.b, df.c).alias('r')).collect() - [Row(r='scala')] - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + Example 4: Flattening an array with mixed types - sc = _get_active_spark_context() - return _invoke_function("elt", _to_seq(sc, inputs, _to_java_column)) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([['a', 'b', 'c'], [1, 2, 3]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show() + +------------------+ + | flatten(data)| + +------------------+ + |[a, b, c, 1, 2, 3]| + +------------------+ + """ + return _invoke_function_over_columns("flatten", col) @_try_remote_functions -def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: +def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Column: """ - Returns the index (1-based) of the given string (`str`) in the comma-delimited - list (`strArray`). Returns 0, if the string was not found or if the given string (`str`) - contains a comma. + Array function: creates an array containing a column repeated count times. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - The given string to be found. - A column that evaluates to a string. - str_array : :class:`~pyspark.sql.Column` or str - The comma-delimited list. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the element to be repeated. + A column of any type. + count : :class:`~pyspark.sql.Column` or str or int + The name of the column, an expression, + or an integer that represents the number of times to repeat the element. + A column that evaluates to an integer. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains an array of repeated elements. + Returns a column that evaluates to an array. + + See Also + -------- + :meth:`pyspark.sql.functions.array` Examples -------- - >>> df = spark.createDataFrame([("ab", "abc,b,ab,c,def")], ['a', 'b']) - >>> df.select(find_in_set(df.a, df.b).alias('r')).collect() - [Row(r=3)] - """ - return _invoke_function_over_columns("find_in_set", str, str_array) + Example 1: Usage with string + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('ab',)], ['data']) + >>> df.select(sf.array_repeat(df.data, 3)).show() + +---------------------+ + |array_repeat(data, 3)| + +---------------------+ + | [ab, ab, ab]| + +---------------------+ -@_try_remote_functions -def like( - str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None -) -> Column: - """ - Returns true if str matches `pattern` with `escape`, - null if any arguments are null, false otherwise. - The default escape character is the '\'. + Example 2: Usage with integer - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(3,)], ['data']) + >>> df.select(sf.array_repeat(df.data, 2)).show() + +---------------------+ + |array_repeat(data, 2)| + +---------------------+ + | [3, 3]| + +---------------------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - A string. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or str - A string. The pattern is a string which is matched literally, with - exception to the following special symbols: - _ matches any one character in the input (similar to . in posix regular expressions) - % matches zero or more characters in the input (similar to .* in posix regular - expressions) - Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order - to match "\abc", the pattern should be "\\abc". - When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back - to Spark 1.6 behavior regarding string literal parsing. For example, if the config is - enabled, the pattern to match "\abc" should be "\abc". - A column that evaluates to a string. - escapeChar : :class:`~pyspark.sql.Column`, optional - An character added since Spark 3.0. The default escape character is the '\'. - If an escape character precedes a special symbol or another escape character, the - following character is matched literally. It is invalid to escape any other character. - A column that evaluates to a string. + Example 3: Usage with array - See Also - -------- - :meth:`pyspark.sql.functions.ilike` - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.regexp_like` + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 'banana'],)], ['data']) + >>> df.select(sf.array_repeat(df.data, 2)).show(truncate=False) + +----------------------------------+ + |array_repeat(data, 2) | + +----------------------------------+ + |[[apple, banana], [apple, banana]]| + +----------------------------------+ - Examples - -------- - >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) - >>> df.select(like(df.a, df.b).alias('r')).collect() - [Row(r=True)] + Example 4: Usage with null - >>> df = spark.createDataFrame( - ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], - ... ['a', 'b'] - ... ) - >>> df.select(like(df.a, df.b, lit('/')).alias('r')).collect() - [Row(r=True)] + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", IntegerType(), True) + ... ]) + >>> df = spark.createDataFrame([(None, )], schema=schema) + >>> df.select(sf.array_repeat(df.data, 3)).show() + +---------------------+ + |array_repeat(data, 3)| + +---------------------+ + | [NULL, NULL, NULL]| + +---------------------+ """ - if escapeChar is not None: - return _invoke_function_over_columns("like", str, pattern, escapeChar) - else: - return _invoke_function_over_columns("like", str, pattern) + count = _enum_to_value(count) + count = lit(count) if isinstance(count, int) else count + + return _invoke_function_over_columns("array_repeat", col, count) @_try_remote_functions -def ilike( - str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None -) -> Column: +def arrays_zip(*cols: "ColumnOrName") -> Column: """ - Returns true if str matches `pattern` with `escape` case-insensitively, - null if any arguments are null, false otherwise. - The default escape character is the '\'. + Array function: Returns a merged array of structs in which the N-th struct contains all + N-th values of input arrays. If one of the arrays is shorter than others then + the resulting struct type value will be a `null` for missing elements. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - A string. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or str - A string. The pattern is a string which is matched literally, with - exception to the following special symbols: - _ matches any one character in the input (similar to . in posix regular expressions) - % matches zero or more characters in the input (similar to .* in posix regular - expressions) - Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order - to match "\abc", the pattern should be "\\abc". - When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back - to Spark 1.6 behavior regarding string literal parsing. For example, if the config is - enabled, the pattern to match "\abc" should be "\abc". - A column that evaluates to a string. - escapeChar : :class:`~pyspark.sql.Column`, optional - An character added since Spark 3.0. The default escape character is the '\'. - If an escape character precedes a special symbol or another escape character, the - following character is matched literally. It is invalid to escape any other character. - A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or str + Columns of arrays to be merged. + A column that evaluates to an array. - See Also - -------- - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.regexp_like` + Returns + ------- + :class:`~pyspark.sql.Column` + Merged array of entries. + Returns a column that evaluates to an array. Examples -------- - >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) - >>> df.select(ilike(df.a, df.b).alias('r')).collect() - [Row(r=True)] + Example 1: Zipping two arrays of the same length - >>> df = spark.createDataFrame( - ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], - ... ['a', 'b'] - ... ) - >>> df.select(ilike(df.a, df.b, lit('/')).alias('r')).collect() - [Row(r=True)] - """ - if escapeChar is not None: - return _invoke_function_over_columns("ilike", str, pattern, escapeChar) - else: - return _invoke_function_over_columns("ilike", str, pattern) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3], ['a', 'b', 'c'])], ['nums', 'letters']) + >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) + +-------------------------+ + |arrays_zip(nums, letters)| + +-------------------------+ + |[{1, a}, {2, b}, {3, c}] | + +-------------------------+ -@_try_remote_functions -def lcase(str: "ColumnOrName") -> Column: - """ - Returns `str` with all characters changed to lowercase. + Example 2: Zipping arrays of different lengths - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2], ['a', 'b', 'c'])], ['nums', 'letters']) + >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) + +---------------------------+ + |arrays_zip(nums, letters) | + +---------------------------+ + |[{1, a}, {2, b}, {NULL, c}]| + +---------------------------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. + Example 3: Zipping more than two arrays - See Also - -------- - :meth:`pyspark.sql.functions.lower` - :meth:`pyspark.sql.functions.ucase` - :meth:`pyspark.sql.functions.upper` + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [([1, 2], ['a', 'b'], [True, False])], ['nums', 'letters', 'bools']) + >>> df.select(sf.arrays_zip(df.nums, df.letters, df.bools)).show(truncate=False) + +--------------------------------+ + |arrays_zip(nums, letters, bools)| + +--------------------------------+ + |[{1, a, true}, {2, b, false}] | + +--------------------------------+ - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.lcase(sf.lit("Spark"))).show() - +------------+ - |lcase(Spark)| - +------------+ - | spark| - +------------+ + Example 4: Zipping arrays with null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, None], ['a', None, 'c'])], ['nums', 'letters']) + >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) + +------------------------------+ + |arrays_zip(nums, letters) | + +------------------------------+ + |[{1, a}, {2, NULL}, {NULL, c}]| + +------------------------------+ """ - return _invoke_function_over_columns("lcase", str) + return _invoke_function_over_seq_of_columns("arrays_zip", cols) @_try_remote_functions -def ucase(str: "ColumnOrName") -> Column: +def sequence( + start: "ColumnOrName", stop: "ColumnOrName", step: Optional["ColumnOrName"] = None +) -> Column: """ - Returns `str` with all characters changed to uppercase. + Array function: Generate a sequence of integers from `start` to `stop`, incrementing by `step`. + If `step` is not set, the function increments by 1 if `start` is less than or equal to `stop`, + otherwise it decrements by 1. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. + start : :class:`~pyspark.sql.Column` or str + The starting value (inclusive) of the sequence. + A column that evaluates to an integral, date, or timestamp. + stop : :class:`~pyspark.sql.Column` or str + The last value (inclusive) of the sequence. + A column that evaluates to an integral, date, or timestamp. + step : :class:`~pyspark.sql.Column` or str, optional + The value to add to the current element to get the next element in the sequence. + The default is 1 if `start` is less than or equal to `stop`, otherwise -1. + A column that evaluates to an integral or interval. - See Also - -------- - :meth:`pyspark.sql.functions.upper` - :meth:`pyspark.sql.functions.lcase` - :meth:`pyspark.sql.functions.lower` + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains an array of sequence values. + Returns a column that evaluates to an array. Examples -------- + Example 1: Generating a sequence with default step + >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.ucase(sf.lit("Spark"))).show() - +------------+ - |ucase(Spark)| - +------------+ - | SPARK| - +------------+ - """ - return _invoke_function_over_columns("ucase", str) + >>> df = spark.createDataFrame([(-2, 2)], ['start', 'stop']) + >>> df.select(sf.sequence(df.start, df.stop)).show() + +---------------------+ + |sequence(start, stop)| + +---------------------+ + | [-2, -1, 0, 1, 2]| + +---------------------+ + Example 2: Generating a sequence with a custom step -@_try_remote_functions -def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: - """ - Returns the leftmost `len`(`len` can be string type) characters from the string `str`, - if `len` is less or equal than 0 the result is an empty string. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(4, -4, -2)], ['start', 'stop', 'step']) + >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() + +---------------------------+ + |sequence(start, stop, step)| + +---------------------------+ + | [4, 2, 0, -2, -4]| + +---------------------------+ - .. versionadded:: 3.5.0 - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string or binary. - len : :class:`~pyspark.sql.Column` or str - Input column or strings, the leftmost `len`. - A column that evaluates to an integer. + Example 3: Generating a sequence with a negative step - Examples - -------- - >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) - >>> df.select(left(df.a, df.b).alias('r')).collect() - [Row(r='Spa')] + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(5, 1, -1)], ['start', 'stop', 'step']) + >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() + +---------------------------+ + |sequence(start, stop, step)| + +---------------------------+ + | [5, 4, 3, 2, 1]| + +---------------------------+ """ - return _invoke_function_over_columns("left", str, len) + if step is None: + return _invoke_function_over_columns("sequence", start, stop) + else: + return _invoke_function_over_columns("sequence", start, stop, step) -@_try_remote_functions -def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: - """ - Returns the rightmost `len`(`len` can be string type) characters from the string `str`, - if `len` is less or equal than 0 the result is an empty string. +# ---------------------- Struct Functions ---------------------- - .. versionadded:: 3.5.0 - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - len : :class:`~pyspark.sql.Column` or str - Input column or strings, the rightmost `len`. - A column that evaluates to an integer. +@overload +def struct(*cols: "ColumnOrName") -> Column: ... - Examples - -------- - >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) - >>> df.select(right(df.a, df.b).alias('r')).collect() - [Row(r='SQL')] - """ - return _invoke_function_over_columns("right", str, len) + +@overload +def struct(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... @_try_remote_functions -def mask( - col: "ColumnOrName", - upperChar: Optional["ColumnOrName"] = None, - lowerChar: Optional["ColumnOrName"] = None, - digitChar: Optional["ColumnOrName"] = None, - otherChar: Optional["ColumnOrName"] = None, +def struct( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], ) -> Column: - """ - Masks the given string value. This can be useful for creating copies of tables with sensitive - information removed. + """Creates a new struct column. - .. versionadded:: 3.5.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col: :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to a string. - upperChar: :class:`~pyspark.sql.Column` or str, optional - character to replace upper-case characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. - lowerChar: :class:`~pyspark.sql.Column` or str, optional - character to replace lower-case characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. - digitChar: :class:`~pyspark.sql.Column` or str, optional - character to replace digit characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. - otherChar: :class:`~pyspark.sql.Column` or str, optional - character to replace all other characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. + cols : list, set, :class:`~pyspark.sql.Column` or column name + column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct. + Each a column of any type. Returns ------- :class:`~pyspark.sql.Column` - Returns a column that evaluates to a string. + a struct type column of given columns. + Returns a column that evaluates to a struct. + + See Also + -------- + :meth:`pyspark.sql.functions.named_struct` Examples -------- - >>> df = spark.createDataFrame([("AbCD123-@$#",), ("abcd-EFGH-8765-4321",)], ['data']) - >>> df.select(mask(df.data).alias('r')).collect() - [Row(r='XxXXnnn-@$#'), Row(r='xxxx-XXXX-nnnn-nnnn')] - >>> df.select(mask(df.data, lit('Y')).alias('r')).collect() - [Row(r='YxYYnnn-@$#'), Row(r='xxxx-YYYY-nnnn-nnnn')] - >>> df.select(mask(df.data, lit('Y'), lit('y')).alias('r')).collect() - [Row(r='YyYYnnn-@$#'), Row(r='yyyy-YYYY-nnnn-nnnn')] - >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d')).alias('r')).collect() - [Row(r='YyYYddd-@$#'), Row(r='yyyy-YYYY-dddd-dddd')] - >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d'), lit('*')).alias('r')).collect() - [Row(r='YyYYddd****'), Row(r='yyyy*YYYY*dddd*dddd')] + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) + >>> df.select("*", sf.struct('age', df.name)).show() + +-----+---+-----------------+ + | name|age|struct(age, name)| + +-----+---+-----------------+ + |Alice| 2| {2, Alice}| + | Bob| 5| {5, Bob}| + +-----+---+-----------------+ """ - - _upperChar = lit("X") if upperChar is None else upperChar - _lowerChar = lit("x") if lowerChar is None else lowerChar - _digitChar = lit("n") if digitChar is None else digitChar - _otherChar = lit(None) if otherChar is None else otherChar - return _invoke_function_over_columns( - "mask", col, _upperChar, _lowerChar, _digitChar, _otherChar - ) + if len(cols) == 1 and isinstance(cols[0], (list, set)): + cols = cols[0] # type: ignore[assignment] + return _invoke_function_over_seq_of_columns("struct", cols) # type: ignore[arg-type] @_try_remote_functions -def collate(col: "ColumnOrName", collation: str) -> Column: +def named_struct(*cols: "ColumnOrName") -> Column: """ - Marks a given column with specified collation. + Creates a struct with the given field names and values. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Target string column to work on. - collation : str - Target collation name. + cols : :class:`~pyspark.sql.Column` or column name + list of columns to work on. Returns ------- :class:`~pyspark.sql.Column` - A new column of string type, where each value has the specified collation. + + See Also + -------- + :meth:`pyspark.sql.functions.struct` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, 2)], ['a', 'b']) + >>> df.select("*", sf.named_struct(sf.lit('x'), df.a, sf.lit('y'), "b")).show() + +---+---+------------------------+ + | a| b|named_struct(x, a, y, b)| + +---+---+------------------------+ + | 1| 2| {1, 2}| + +---+---+------------------------+ """ - from pyspark.sql.classic.column import _to_java_column + return _invoke_function_over_seq_of_columns("named_struct", cols) - return _invoke_function("collate", _to_java_column(col), _enum_to_value(collation)) + +# ---------------------- Map Functions ---------------------- + + +@overload +def create_map(*cols: "ColumnOrName") -> Column: ... + + +@overload +def create_map(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... @_try_remote_functions -def collation(col: "ColumnOrName") -> Column: +def create_map( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], +) -> Column: """ - Returns the collation name of a given column. + Map function: Creates a new map column from an even number of input columns or + column references. The input columns are grouped into key-value pairs to form a map. + For instance, the input (key1, value1, key2, value2, ...) would produce a map that + associates key1 with value1, key2 with value2, and so on. The function supports + grouping columns as a list as well. - .. versionadded:: 4.0.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - Target string column to work on. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - collation name of a given expression. - Returns a column that evaluates to a string. - - Examples - -------- - >>> df = spark.createDataFrame([('name',)], ['dt']) - >>> df.select(collation('dt').alias('collation')).show(truncate=False) - +--------------------------+ - |collation | - +--------------------------+ - |SYSTEM.BUILTIN.UTF8_BINARY| - +--------------------------+ - """ - return _invoke_function_over_columns("collation", col) - - -@_try_remote_functions -def quote(col: "ColumnOrName") -> Column: - r"""Returns `str` enclosed by single quotes and each instance of - single quote in it is preceded by a backslash. - - .. versionadded:: 4.1.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to be quoted. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - quoted string - Returns a column that evaluates to a string. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Don't"], "STRING") - >>> df.select("*", sf.quote("value")).show() - +-----+------------+ - |value|quote(value)| - +-----+------------+ - |Don't| 'Don\'t'| - +-----+------------+ - """ - return _invoke_function_over_columns("quote", col) - - -# ---------------------- Collection functions ------------------------------ - - -@overload -def create_map(*cols: "ColumnOrName") -> Column: ... - - -@overload -def create_map(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... - - -@_try_remote_functions -def create_map( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], -) -> Column: - """ - Map function: Creates a new map column from an even number of input columns or - column references. The input columns are grouped into key-value pairs to form a map. - For instance, the input (key1, value1, key2, value2, ...) would produce a map that - associates key1 with value1, key2 with value2, and so on. The function supports - grouping columns as a list as well. - - .. versionadded:: 2.0.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- @@ -20113,119 +19774,147 @@ def map_from_arrays(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: return _invoke_function_over_columns("map_from_arrays", col1, col2) -@overload -def array(*cols: "ColumnOrName") -> Column: ... +@_try_remote_functions +def map_contains_key(col: "ColumnOrName", value: Any) -> Column: + """ + Map function: Returns true if the map contains the key. + .. versionadded:: 3.4.0 -@overload -def array(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the map. + value : + A literal value, or a :class:`~pyspark.sql.Column` expression. + + .. versionchanged:: 4.0.0 + `value` now also accepts a Column type. + + Returns + ------- + :class:`~pyspark.sql.Column` + True if key is in the map and False otherwise. + + Examples + -------- + Example 1: The key is in the map + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.map_contains_key("data", 1)).show() + +-------------------------+ + |map_contains_key(data, 1)| + +-------------------------+ + | true| + +-------------------------+ + + Example 2: The key is not in the map + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.map_contains_key("data", -1)).show() + +--------------------------+ + |map_contains_key(data, -1)| + +--------------------------+ + | false| + +--------------------------+ + + Example 3: Check for key using a column + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data, 1 as key") + >>> df.select(sf.map_contains_key("data", sf.col("key"))).show() + +---------------------------+ + |map_contains_key(data, key)| + +---------------------------+ + | true| + +---------------------------+ + """ + return _invoke_function_over_columns("map_contains_key", col, lit(value)) @_try_remote_functions -def array( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], -) -> Column: +def map_keys(col: "ColumnOrName") -> Column: """ - Collection function: Creates a new array column from the input columns or column names. + Map function: Returns an unordered array containing the keys of the map. - .. versionadded:: 1.4.0 + .. versionadded:: 2.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - Column names or :class:`~pyspark.sql.Column` objects that have the same data type. + col : :class:`~pyspark.sql.Column` or str + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - A new Column of array type, where each value is an array containing the corresponding values - from the input columns. - Returns a column that evaluates to an array. + Keys of the map as an array. Examples -------- - Example 1: Basic usage of array function with column names. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array('name', 'occupation')).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, doctor]| - | [Bob, engineer]| - +-----------------------+ - - Example 2: Usage of array function with Column objects. + Example 1: Extracting keys from a simple map >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array(df.name, df.occupation)).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, doctor]| - | [Bob, engineer]| - +-----------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_keys("data"))).show() + +--------------------------------+ + |sort_array(map_keys(data), true)| + +--------------------------------+ + | [1, 2]| + +--------------------------------+ - Example 3: Single argument as list of column names. + Example 2: Extracting keys from a map with complex keys >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array(['name', 'occupation'])).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, doctor]| - | [Bob, engineer]| - +-----------------------+ + >>> df = spark.sql("SELECT map(array(1, 2), 'a', array(3, 4), 'b') as data") + >>> df.select(sf.sort_array(sf.map_keys("data"))).show() + +--------------------------------+ + |sort_array(map_keys(data), true)| + +--------------------------------+ + | [[1, 2], [3, 4]]| + +--------------------------------+ - Example 4: Usage of array function with columns of different types. + Example 3: Extracting keys from a map with duplicate keys >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("Alice", 2, 22.2), ("Bob", 5, 36.1)], - ... ("name", "age", "weight")) - >>> df.select(sf.array(['age', 'weight'])).show() - +------------------+ - |array(age, weight)| - +------------------+ - | [2.0, 22.2]| - | [5.0, 36.1]| - +------------------+ + >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") + >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") + >>> df.select(sf.map_keys("data")).show() + +--------------+ + |map_keys(data)| + +--------------+ + | [1]| + +--------------+ + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) - Example 5: array function with a column containing null values. + Example 4: Extracting keys from an empty map >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", None), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array('name', 'occupation')).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, NULL]| - | [Bob, engineer]| - +-----------------------+ + >>> df = spark.sql("SELECT map() as data") + >>> df.select(sf.map_keys("data")).show() + +--------------+ + |map_keys(data)| + +--------------+ + | []| + +--------------+ """ - if len(cols) == 1 and isinstance(cols[0], (list, set)): - cols = cols[0] # type: ignore[assignment] - return _invoke_function_over_seq_of_columns("array", cols) # type: ignore[arg-type] + return _invoke_function_over_columns("map_keys", col) @_try_remote_functions -def array_contains(col: "ColumnOrName", value: Any) -> Column: +def map_values(col: "ColumnOrName") -> Column: """ - Collection function: Returns true if the array contains the value, false if not. Returns - null if the array or value is null, or if the value is not found and the array contains a - null element. + Map function: Returns an unordered array containing the values of the map. - .. versionadded:: 1.5.0 + .. versionadded:: 2.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -20233,176 +19922,152 @@ def array_contains(col: "ColumnOrName", value: Any) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - The target column containing the arrays. - A column that evaluates to an array. - value : - The value or column to check for in the array. - A column of the same type as the array elements. + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - A new Column of Boolean type, where each value indicates whether the corresponding array - from the input column contains the specified value. - Returns a column that evaluates to a boolean. - - See Also - -------- - :meth:`pyspark.sql.functions.array_position` + Values of the map as an array. Examples -------- - Example 1: Basic usage of array_contains function. + Example 1: Extracting values from a simple map >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],), ([],)], ['data']) - >>> df.select(sf.array_contains(df.data, "a")).show() - +-----------------------+ - |array_contains(data, a)| - +-----------------------+ - | true| - | false| - +-----------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_values("data"))).show() + +----------------------------------+ + |sort_array(map_values(data), true)| + +----------------------------------+ + | [a, b]| + +----------------------------------+ - Example 2: Usage of array_contains function with a column. + Example 2: Extracting values from a map with complex values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], "c"), - ... (["c", "d", "e"], "d"), - ... (["e", "a", "c"], "b")], ["data", "item"]) - >>> df.select(sf.array_contains(df.data, sf.col("item"))).show() - +--------------------------+ - |array_contains(data, item)| - +--------------------------+ - | true| - | true| - | false| - +--------------------------+ + >>> df = spark.sql("SELECT map(1, array('a', 'b'), 2, array('c', 'd')) as data") + >>> df.select(sf.sort_array(sf.map_values("data"))).show() + +----------------------------------+ + |sort_array(map_values(data), true)| + +----------------------------------+ + | [[a, b], [c, d]]| + +----------------------------------+ - Example 3: Attempt to use array_contains function with a null array. + Example 3: Extracting values from a map with null values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), (["a", "b", "c"],)], ['data']) - >>> df.select(sf.array_contains(df.data, "a")).show() - +-----------------------+ - |array_contains(data, a)| - +-----------------------+ - | NULL| - | true| - +-----------------------+ + >>> df = spark.sql("SELECT map(1, null, 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_values("data"))).show() + +----------------------------------+ + |sort_array(map_values(data), true)| + +----------------------------------+ + | [NULL, b]| + +----------------------------------+ - Example 4: Usage of array_contains with an array column containing null values. + Example 4: Extracting values from a map with duplicate values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_contains(df.data, "a")).show() - +-----------------------+ - |array_contains(data, a)| - +-----------------------+ - | true| - +-----------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'a') as data") + >>> df.select(sf.map_values("data")).show() + +----------------+ + |map_values(data)| + +----------------+ + | [a, a]| + +----------------+ - Example 5: Value absent from an array that contains a null element returns NULL. + Example 5: Extracting values from an empty map >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_contains(df.data, "b")).show() - +-----------------------+ - |array_contains(data, b)| - +-----------------------+ - | NULL| - +-----------------------+ + >>> df = spark.sql("SELECT map() as data") + >>> df.select(sf.map_values("data")).show() + +----------------+ + |map_values(data)| + +----------------+ + | []| + +----------------+ """ - return _invoke_function_over_columns("array_contains", col, lit(value)) + return _invoke_function_over_columns("map_values", col) @_try_remote_functions -def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column: +def map_entries(col: "ColumnOrName") -> Column: """ - Collection function: This function returns a boolean column indicating if the input arrays - have common non-null elements, returning true if they do, null if the arrays do not contain - any common elements but are not empty and at least one of them contains a null element, - and false otherwise. + Map function: Returns an unordered array of all entries in the given map. - .. versionadded:: 2.4.0 + .. versionadded:: 3.0.0 .. versionchanged:: 3.4.0 - Supports Spark Connect. + Spark Connect. Parameters ---------- - a1, a2 : :class:`~pyspark.sql.Column` or str - The names of the columns that contain the input arrays. - Each a column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or str + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - A new Column of Boolean type, where each value indicates whether the corresponding arrays - from the input columns contain any common elements. - Returns a column that evaluates to a boolean. + An array of key value pairs as a struct type Examples -------- - Example 1: Basic usage of arrays_overlap function. + Example 1: Extracting entries from a simple map >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b"], ["b", "c"]), (["a"], ["b", "c"])], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | true| - | false| - +--------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_entries("data"))).show() + +-----------------------------------+ + |sort_array(map_entries(data), true)| + +-----------------------------------+ + | [{1, a}, {2, b}]| + +-----------------------------------+ - Example 2: Usage of arrays_overlap function with arrays containing null elements. + Example 2: Extracting entries from a map with complex keys and values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None], ["b", None]), (["a"], ["b", "c"])], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | NULL| - | false| - +--------------------+ + >>> df = spark.sql("SELECT map(array(1, 2), array('a', 'b'), " + ... "array(3, 4), array('c', 'd')) as data") + >>> df.select(sf.sort_array(sf.map_entries("data"))).show(truncate=False) + +------------------------------------+ + |sort_array(map_entries(data), true) | + +------------------------------------+ + |[{[1, 2], [a, b]}, {[3, 4], [c, d]}]| + +------------------------------------+ - Example 3: Usage of arrays_overlap function with arrays that are null. + Example 3: Extracting entries from a map with duplicate keys >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None, ["b", "c"]), (["a"], None)], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | NULL| - | NULL| - +--------------------+ + >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") + >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") + >>> df.select(sf.map_entries("data")).show() + +-----------------+ + |map_entries(data)| + +-----------------+ + | [{1, b}]| + +-----------------+ + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) - Example 4: Usage of arrays_overlap on arrays with identical elements. + Example 4: Extracting entries from an empty map >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b"], ["a", "b"]), (["a"], ["a"])], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | true| - | true| - +--------------------+ + >>> df = spark.sql("SELECT map() as data") + >>> df.select(sf.map_entries("data")).show() + +-----------------+ + |map_entries(data)| + +-----------------+ + | []| + +-----------------+ """ - return _invoke_function_over_columns("arrays_overlap", a1, a2) + return _invoke_function_over_columns("map_entries", col) @_try_remote_functions -def slice( - x: "ColumnOrName", start: Union["ColumnOrName", int], length: Union["ColumnOrName", int] -) -> Column: +def map_from_entries(col: "ColumnOrName") -> Column: """ - Array function: Returns a new array column by slicing the input array column from - a start index to a specific length. The indices start at 1, and can be negative to index - from the end of the array. The length specifies the number of elements in the resulting array. + Map function: Transforms an array of key-value pair entries (structs with two fields) + into a map. The first field of each entry is used as the key and the second field + as the value in the resulting map column .. versionadded:: 2.4.0 @@ -20411,1778 +20076,1587 @@ def slice( Parameters ---------- - x : :class:`~pyspark.sql.Column` or str - Input array column or column name to be sliced. - A column that evaluates to an array. - start : :class:`~pyspark.sql.Column`, str, or int - The start index for the slice operation. If negative, starts the index from the - end of the array. - A column that evaluates to an integer. - length : :class:`~pyspark.sql.Column`, str, or int - The length of the slice, representing number of elements in the resulting array. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - A new Column object of Array type, where each value is a slice of the corresponding - list from the input column. - Returns a column that evaluates to an array. + A map created from the given array of entries. Examples -------- - Example 1: Basic usage of the slice function. + Example 1: Basic usage of map_from_entries >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) - >>> df.select(sf.slice(df.x, 2, 2)).show() - +--------------+ - |slice(x, 2, 2)| - +--------------+ - | [2, 3]| - | [5]| - +--------------+ + >>> df = spark.sql("SELECT array(struct(1, 'a'), struct(2, 'b')) as data") + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {1 -> a, 2 -> b}| + +----------------------+ - Example 2: Slicing with negative start index. + Example 2: map_from_entries with null values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) - >>> df.select(sf.slice(df.x, -1, 1)).show() - +---------------+ - |slice(x, -1, 1)| - +---------------+ - | [3]| - | [5]| - +---------------+ + >>> df = spark.sql("SELECT array(struct(1, null), struct(2, 'b')) as data") + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {1 -> NULL, 2 -> b}| + +----------------------+ - Example 3: Slice function with column inputs for start and length. + Example 3: map_from_entries with a DataFrame + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([([Row(1, "a"), Row(2, "b")],), ([Row(3, "c")],)], ['data']) + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {1 -> a, 2 -> b}| + | {3 -> c}| + +----------------------+ + + Example 4: map_from_entries with empty array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3], 2, 2), ([4, 5], 1, 3)], ['x', 'start', 'length']) - >>> df.select(sf.slice(df.x, df.start, df.length)).show() - +-----------------------+ - |slice(x, start, length)| - +-----------------------+ - | [2, 3]| - | [4, 5]| - +-----------------------+ + >>> from pyspark.sql.types import ArrayType, StringType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType( + ... StructType([ + ... StructField("key", IntegerType()), + ... StructField("value", StringType()) + ... ]) + ... ), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {}| + +----------------------+ """ - start = _enum_to_value(start) - start = lit(start) if isinstance(start, int) else start - length = _enum_to_value(length) - length = lit(length) if isinstance(length, int) else length + return _invoke_function_over_columns("map_from_entries", col) - return _invoke_function_over_columns("slice", x, start, length) + +@overload +def map_concat(*cols: "ColumnOrName") -> Column: ... + + +@overload +def map_concat(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... @_try_remote_functions -def trim_array(x: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: +def map_concat( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], +) -> Column: """ - Array function: Returns the given array column with the last ``n`` elements removed. - Raises an error if ``n`` is negative or greater than the number of elements in the array. + Map function: Returns the union of all given maps. - .. versionadded:: 4.4.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - x : :class:`~pyspark.sql.Column` or str - Input array column or column name to be trimmed. - A column that evaluates to an array. - n : :class:`~pyspark.sql.Column`, str, or int - The number of elements to remove from the end of the array. Must be between 0 and - the number of elements in the array (inclusive). - A column that evaluates to an integer. + cols : :class:`~pyspark.sql.Column` or str + Column names or :class:`~pyspark.sql.Column` Returns ------- :class:`~pyspark.sql.Column` - A new Column object of Array type, where each value is the corresponding input array - with its last ``n`` elements removed. - Returns a column that evaluates to an array. + A map of merged entries from other maps. + + Notes + ----- + For duplicate keys in input maps, the handling is governed by `spark.sql.mapKeyDedupPolicy`. + By default, it throws an exception. If set to `LAST_WIN`, it uses the last map's value. Examples -------- - Example 1: Basic usage of the trim_array function. + Example 1: Basic usage of map_concat >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 4, 5],), ([4, 5],)], ['x']) - >>> df.select(sf.trim_array(df.x, 2)).show() - +----------------+ - |trim_array(x, 2)| - +----------------+ - | [1, 2, 3]| - | []| - +----------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +------------------------+ + |map_concat(map1, map2) | + +------------------------+ + |{1 -> a, 2 -> b, 3 -> c}| + +------------------------+ - Example 2: trim_array function with a column input for n. + Example 2: map_concat with overlapping keys >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 4, 5], 1), ([4, 5], 0)], ['x', 'n']) - >>> df.select(sf.trim_array(df.x, df.n)).show() - +----------------+ - |trim_array(x, n)| - +----------------+ - | [1, 2, 3, 4]| - | [4, 5]| - +----------------+ + >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(2, 'c', 3, 'd') as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +------------------------+ + |map_concat(map1, map2) | + +------------------------+ + |{1 -> a, 2 -> c, 3 -> d}| + +------------------------+ + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) + + Example 3: map_concat with three maps + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a') as map1, map(2, 'b') as map2, map(3, 'c') as map3") + >>> df.select(sf.map_concat("map1", "map2", "map3")).show(truncate=False) + +----------------------------+ + |map_concat(map1, map2, map3)| + +----------------------------+ + |{1 -> a, 2 -> b, 3 -> c} | + +----------------------------+ + + Example 4: map_concat with empty map + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map() as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +----------------------+ + |map_concat(map1, map2)| + +----------------------+ + |{1 -> a, 2 -> b} | + +----------------------+ + + Example 5: map_concat with null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, null) as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +---------------------------+ + |map_concat(map1, map2) | + +---------------------------+ + |{1 -> a, 2 -> b, 3 -> NULL}| + +---------------------------+ """ - n = _enum_to_value(n) - n = lit(n) if isinstance(n, int) else n - return _invoke_function_over_columns("trim_array", x, n) + if len(cols) == 1 and isinstance(cols[0], (list, set)): + cols = cols[0] # type: ignore[assignment] + return _invoke_function_over_seq_of_columns("map_concat", cols) # type: ignore[arg-type] @_try_remote_functions -def array_join( - col: "ColumnOrName", delimiter: str, null_replacement: Optional[str] = None +def str_to_map( + text: "ColumnOrName", + pairDelim: Optional["ColumnOrName"] = None, + keyValueDelim: Optional["ColumnOrName"] = None, ) -> Column: """ - Array function: Returns a string column by concatenating the elements of the input - array column using the delimiter. Null values within the array can be replaced with - a specified string through the null_replacement argument. If null_replacement is - not set, null values are ignored. - - .. versionadded:: 2.4.0 + Map function: Converts a string into a map after splitting the text into key/value pairs + using delimiters. Both `pairDelim` and `keyValueDelim` are treated as regular expressions. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The input column containing the arrays to be joined. - A column that evaluates to an array. - delimiter : str - The string to be used as the delimiter when joining the array elements. + text : :class:`~pyspark.sql.Column` or str + Input column or strings. A column that evaluates to a string. - null_replacement : str, optional - The string to replace null values within the array. If not set, null values are ignored. + pairDelim : :class:`~pyspark.sql.Column` or str, optional + Delimiter to use to split pairs. Default is comma (,). + A column that evaluates to a string. + keyValueDelim : :class:`~pyspark.sql.Column` or str, optional + Delimiter to use to split key/value. Default is colon (:). A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new column of string type, where each value is the result of joining the corresponding - array from the input column. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.concat` - :meth:`pyspark.sql.functions.concat_ws` + A new column of map type where each string in the original column is converted into a map. + Returns a column that evaluates to a map. Examples -------- - Example 1: Basic usage of array_join function. + Example 1: Using default delimiters >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],), (["a", "b"],)], ['data']) - >>> df.select(sf.array_join(df.data, ",")).show() - +-------------------+ - |array_join(data, ,)| - +-------------------+ - | a,b,c| - | a,b| - +-------------------+ + >>> df = spark.createDataFrame([("a:1,b:2,c:3",)], ["e"]) + >>> df.select(sf.str_to_map(df.e)).show(truncate=False) + +------------------------+ + |str_to_map(e, ,, :) | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + +------------------------+ - Example 2: Usage of array_join function with null_replacement argument. + Example 2: Using custom delimiters >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_join(df.data, ",", "NULL")).show() - +-------------------------+ - |array_join(data, ,, NULL)| - +-------------------------+ - | a,NULL,c| - +-------------------------+ + >>> df = spark.createDataFrame([("a=1;b=2;c=3",)], ["e"]) + >>> df.select(sf.str_to_map(df.e, sf.lit(";"), sf.lit("="))).show(truncate=False) + +------------------------+ + |str_to_map(e, ;, =) | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + +------------------------+ - Example 3: Usage of array_join function without null_replacement argument. + Example 3: Using different delimiters for different rows >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_join(df.data, ",")).show() - +-------------------+ - |array_join(data, ,)| - +-------------------+ - | a,c| - +-------------------+ + >>> df = spark.createDataFrame([("a:1,b:2,c:3",), ("d=4;e=5;f=6",)], ["e"]) + >>> df.select(sf.str_to_map(df.e, + ... sf.when(df.e.contains(";"), sf.lit(";")).otherwise(sf.lit(",")), + ... sf.when(df.e.contains("="), sf.lit("=")).otherwise(sf.lit(":"))).alias("str_to_map") + ... ).show(truncate=False) + +------------------------+ + |str_to_map | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + |{d -> 4, e -> 5, f -> 6}| + +------------------------+ - Example 4: Usage of array_join function with an array that is null. + Example 4: Using a column of delimiters >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([(None,)], schema) - >>> df.select(sf.array_join(df.data, ",")).show() - +-------------------+ - |array_join(data, ,)| - +-------------------+ - | NULL| - +-------------------+ + >>> df = spark.createDataFrame([("a:1,b:2,c:3", ","), ("d=4;e=5;f=6", ";")], ["e", "delim"]) + >>> df.select(sf.str_to_map(df.e, df.delim, sf.lit(":"))).show(truncate=False) + +---------------------------------------+ + |str_to_map(e, delim, :) | + +---------------------------------------+ + |{a -> 1, b -> 2, c -> 3} | + |{d=4 -> NULL, e=5 -> NULL, f=6 -> NULL}| + +---------------------------------------+ - Example 5: Usage of array_join function with an array containing only null values. + Example 5: Using a column of key/value delimiters >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([([None, None],)], schema) - >>> df.select(sf.array_join(df.data, ",", "NULL")).show() - +-------------------------+ - |array_join(data, ,, NULL)| - +-------------------------+ - | NULL,NULL| - +-------------------------+ + >>> df = spark.createDataFrame([("a:1,b:2,c:3", ":"), ("d=4;e=5;f=6", "=")], ["e", "delim"]) + >>> df.select(sf.str_to_map(df.e, sf.lit(","), df.delim)).show(truncate=False) + +------------------------+ + |str_to_map(e, ,, delim) | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + |{d -> 4;e=5;f=6} | + +------------------------+ """ - from pyspark.sql.classic.column import _to_java_column + if pairDelim is None: + pairDelim = lit(",") + if keyValueDelim is None: + keyValueDelim = lit(":") + return _invoke_function_over_columns("str_to_map", text, pairDelim, keyValueDelim) - _get_active_spark_context() - if null_replacement is None: - return _invoke_function("array_join", _to_java_column(col), _enum_to_value(delimiter)) - else: - return _invoke_function( - "array_join", - _to_java_column(col), - _enum_to_value(delimiter), - _enum_to_value(null_replacement), - ) + +# ---------------------- Aggregate Functions ---------------------- @_try_remote_functions -def concat(*cols: "ColumnOrName") -> Column: +def try_avg(col: "ColumnOrName") -> Column: """ - Collection function: Concatenates multiple input columns together into a single column. - The function works with strings, numeric, binary and compatible array columns. - - .. versionadded:: 1.5.0 + Returns the mean calculated from values of a group and the result is null on overflow. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - target column or columns to work on. - Each a column that evaluates to a string, numeric, binary, or array. - - Returns - ------- - :class:`~pyspark.sql.Column` - concatenated values. Type of the `Column` depends on input columns' type. - Returns a column of the same type as the input. - - See Also - -------- - :meth:`pyspark.sql.functions.concat_ws` - :meth:`pyspark.sql.functions.array_join` : to concatenate string columns with delimiter + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- - Example 1: Concatenating string columns + Example 1: Calculating the average age - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd']) - >>> df.select(sf.concat(df.s, df.d)).show() + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) + >>> df.select(sf.try_avg("age")).show() +------------+ - |concat(s, d)| + |try_avg(age)| +------------+ - | abcd123| + | 8.5| +------------+ - Example 2: Concatenating array columns + Example 2: Calculating the average age with None - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2], [3, 4], [5]), ([1, 2], None, [3])], ['a', 'b', 'c']) - >>> df.select(sf.concat(df.a, df.b, df.c)).show() + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.try_avg("age")).show() + +------------+ + |try_avg(age)| + +------------+ + | 3.0| + +------------+ + + Example 3: Overflow results in NULL when ANSI mode is on + + >>> from decimal import Decimal + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.createDataFrame( + ... [(Decimal("1" * 38),), (Decimal(0),)], "number DECIMAL(38, 0)") + ... df.select(sf.try_avg(df.number)).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) +---------------+ - |concat(a, b, c)| + |try_avg(number)| +---------------+ - |[1, 2, 3, 4, 5]| | NULL| +---------------+ + """ + return _invoke_function_over_columns("try_avg", col) - Example 3: Concatenating numeric columns - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) - >>> df.select(sf.concat(df.a, df.b, df.c)).show() - +---------------+ - |concat(a, b, c)| - +---------------+ - | 123| - +---------------+ +@_try_remote_functions +def try_sum(col: "ColumnOrName") -> Column: + """ + Returns the sum calculated from values of a group and the result is null on overflow. - Example 4: Concatenating binary columns + .. versionadded:: 3.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. + + Examples + -------- + Example 1: Calculating the sum of values in a column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytearray(b'abc'), bytearray(b'def'))], ['a', 'b']) - >>> df.select(sf.concat(df.a, df.b)).show() - +-------------------+ - | concat(a, b)| - +-------------------+ - |[61 62 63 64 65 66]| - +-------------------+ + >>> spark.range(10).select(sf.try_sum("id")).show() + +-----------+ + |try_sum(id)| + +-----------+ + | 45| + +-----------+ - Example 5: Concatenating mixed types of columns + Example 2: Using a plus expression together to calculate the sum >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,"abc",3,"def")], ['a','b','c','d']) - >>> df.select(sf.concat(df.a, df.b, df.c, df.d)).show() - +------------------+ - |concat(a, b, c, d)| - +------------------+ - | 1abc3def| - +------------------+ + >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) + >>> df.select(sf.try_sum(sf.col("A") + sf.col("B"))).show() + +----------------+ + |try_sum((A + B))| + +----------------+ + | 10| + +----------------+ + + Example 3: Calculating the summation of ages with None + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.try_sum("age")).show() + +------------+ + |try_sum(age)| + +------------+ + | 6| + +------------+ + + Example 4: Overflow results in NULL when ANSI mode is on + + >>> from decimal import Decimal + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.createDataFrame([(Decimal("1" * 38),)] * 10, "number DECIMAL(38, 0)") + ... df.select(sf.try_sum(df.number)).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +---------------+ + |try_sum(number)| + +---------------+ + | NULL| + +---------------+ """ - return _invoke_function_over_seq_of_columns("concat", cols) + return _invoke_function_over_columns("try_sum", col) @_try_remote_functions -def array_position(col: "ColumnOrName", value: Any) -> Column: +def mode(col: "ColumnOrName", deterministic: bool = False) -> Column: """ - Array function: Locates the position of the first occurrence of the given value - in the given array. Returns null if either of the arguments are null. - - .. versionadded:: 2.4.0 + Returns the most frequent value in a group. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.4.0 - Notes - ----- - The position is not zero based, but 1 based index. Returns 0 if the given - value could not be found in the array. + .. versionchanged:: 4.0.0 + Supports deterministic argument. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target column to work on. - A column that evaluates to an array. - value : Any - value or a :class:`~pyspark.sql.Column` expression to look for. - A column of the same type as the array elements. - - .. versionchanged:: 4.0.0 - `value` now also accepts a Column type. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column of any type. + deterministic : bool, optional + if there are multiple equally-frequent results then return the lowest (defaults to false). + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - position of the value in the given array if found and 0 otherwise. - Returns a column that evaluates to a long. + the most frequent value in a group. - See Also - -------- - :meth:`pyspark.sql.functions.array_contains` + Notes + ----- + Supports Spark Connect. Examples -------- - Example 1: Finding the position of a string in an array of strings - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) - >>> df.select(sf.array_position(df.data, "a")).show() - +-----------------------+ - |array_position(data, a)| - +-----------------------+ - | 3| - +-----------------------+ - - Example 2: Finding the position of a string in an empty array - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_position(df.data, "a")).show() - +-----------------------+ - |array_position(data, a)| - +-----------------------+ - | 0| - +-----------------------+ - - Example 3: Finding the position of an integer in an array of integers - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_position(df.data, 2)).show() - +-----------------------+ - |array_position(data, 2)| - +-----------------------+ - | 2| - +-----------------------+ - - Example 4: Finding the position of a non-existing value in an array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) - >>> df.select(sf.array_position(df.data, "d")).show() - +-----------------------+ - |array_position(data, d)| - +-----------------------+ - | 0| - +-----------------------+ - - Example 5: Finding the position of a value in an array with nulls - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([None, "b", "a"],)], ['data']) - >>> df.select(sf.array_position(df.data, "a")).show() - +-----------------------+ - |array_position(data, a)| - +-----------------------+ - | 3| - +-----------------------+ + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.mode("year")).sort("course").show() + +------+----------+ + |course|mode(year)| + +------+----------+ + | Java| 2012| + |dotNET| 2012| + +------+----------+ - Example 6: Finding the position of a column's value in an array of integers + When multiple values have the same greatest frequency then either any of values is returned if + deterministic is false or is not defined, or the lowest value is returned if deterministic is + true. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([10, 20, 30], 20)], ['data', 'col']) - >>> df.select(sf.array_position(df.data, df.col)).show() - +-------------------------+ - |array_position(data, col)| - +-------------------------+ - | 2| - +-------------------------+ + >>> df = spark.createDataFrame([(-10,), (0,), (10,)], ["col"]) + >>> df.select(sf.mode("col", False)).show() # doctest: +SKIP + +---------+ + |mode(col)| + +---------+ + | 0| + +---------+ + >>> df.select(sf.mode("col", True)).show() + +---------------------------------------+ + |mode() WITHIN GROUP (ORDER BY col DESC)| + +---------------------------------------+ + | -10| + +---------------------------------------+ """ - return _invoke_function_over_columns("array_position", col, lit(value)) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("mode", _to_java_column(col), _enum_to_value(deterministic)) @_try_remote_functions -def element_at(col: "ColumnOrName", extraction: Any) -> Column: +def max(col: "ColumnOrName") -> Column: """ - Collection function: - (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will - throw an error. If index < 0, accesses elements from the last to the first. - If 'spark.sql.ansi.enabled' is set to true, an exception will be thrown if the index is out - of array boundaries instead of returning NULL. - - (map, key) - Returns value for given key in `extraction` if col is map. The function always - returns NULL if the key is not contained in the map. + Aggregate function: returns the maximum value of the expression in a group. - .. versionadded:: 2.4.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing array or map. - A column that evaluates to an array or map. - extraction : - index to check for in array or key to check for in map. - A column that evaluates to an integer for an array, or the key type for a map. + col : :class:`~pyspark.sql.Column` or column name + The target column on which the maximum value is computed. Returns ------- :class:`~pyspark.sql.Column` - value at given position. - Returns a column of the element type of the input array, or the value type of the input map. - - Notes - ----- - The position is not zero based, but 1 based index. - If extraction is a string, :meth:`element_at` treats it as a literal string, - while :meth:`try_element_at` treats it as a column name. + A column that contains the maximum value computed. See Also -------- - :meth:`pyspark.sql.functions.get` - :meth:`pyspark.sql.functions.try_element_at` + :meth:`pyspark.sql.functions.min` + :meth:`pyspark.sql.functions.avg` + :meth:`pyspark.sql.functions.sum` + + Notes + ----- + - Null values are ignored during the computation. + - NaN values are larger than any other numeric value. Examples -------- - Example 1: Getting the first element of an array + Example 1: Compute the maximum value of a numeric column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.element_at(df.data, 1)).show() - +-------------------+ - |element_at(data, 1)| - +-------------------+ - | a| - +-------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.range(10) + >>> df.select(sf.max(df.id)).show() + +-------+ + |max(id)| + +-------+ + | 9| + +-------+ - Example 2: Getting the last element of an array using negative index + Example 2: Compute the maximum value of a string column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.element_at(df.data, -1)).show() - +--------------------+ - |element_at(data, -1)| - +--------------------+ - | c| - +--------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("A",), ("B",), ("C",)], ["value"]) + >>> df.select(sf.max(df.value)).show() + +----------+ + |max(value)| + +----------+ + | C| + +----------+ - Example 3: Getting a value from a map using a key + Example 3: Compute the maximum value of a column in a grouped DataFrame - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.element_at(df.data, sf.lit("a"))).show() - +-------------------+ - |element_at(data, a)| - +-------------------+ - | 1.0| - +-------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("A", 1), ("A", 2), ("B", 3), ("B", 4)], ["key", "value"]) + >>> df.groupBy("key").agg(sf.max(df.value)).show() + +---+----------+ + |key|max(value)| + +---+----------+ + | A| 2| + | B| 4| + +---+----------+ - Example 4: Getting a non-existing value from a map using a key + Example 4: Compute the maximum value of multiple columns in a grouped DataFrame - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.element_at(df.data, sf.lit("c"))).show() - +-------------------+ - |element_at(data, c)| - +-------------------+ - | NULL| - +-------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame( + ... [("A", 1, 2), ("A", 2, 3), ("B", 3, 4), ("B", 4, 5)], ["key", "value1", "value2"]) + >>> df.groupBy("key").agg(sf.max("value1"), sf.max("value2")).show() + +---+-----------+-----------+ + |key|max(value1)|max(value2)| + +---+-----------+-----------+ + | A| 2| 3| + | B| 4| 5| + +---+-----------+-----------+ - Example 5: Getting a value from a map using a literal string as the key + Example 5: Compute the maximum value of a column with null values - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) - >>> df.select(sf.element_at(df.data, 'b')).show() - +-------------------+ - |element_at(data, b)| - +-------------------+ - | 2.0| - +-------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1,), (2,), (None,)], ["value"]) + >>> df.select(sf.max(df.value)).show() + +----------+ + |max(value)| + +----------+ + | 2| + +----------+ + + Example 6: Compute the maximum value of a column with "NaN" values + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1.1,), (float("nan"),), (3.3,)], ["value"]) + >>> df.select(sf.max(df.value)).show() + +----------+ + |max(value)| + +----------+ + | NaN| + +----------+ """ - return _invoke_function_over_columns("element_at", col, lit(extraction)) + return _invoke_function_over_columns("max", col) @_try_remote_functions -def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: +def min(col: "ColumnOrName") -> Column: """ - Collection function: - (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will - throw an error. If index < 0, accesses elements from the last to the first. The function - always returns NULL if the index exceeds the length of the array. + Aggregate function: returns the minimum value of the expression in a group. - (map, key) - Returns value for given key. The function always returns NULL if the key is not - contained in the map. + .. versionadded:: 1.3.0 - .. versionadded:: 3.5.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing array or map. - A column that evaluates to an array or map. - extraction : - index to check for in array or key to check for in map. - A column that evaluates to an integer for an array, or the key type for a map. + col : :class:`~pyspark.sql.Column` or column name + The target column on which the minimum value is computed. Returns ------- :class:`~pyspark.sql.Column` - Returns a column of the element type of the input array, or the value type of the input map. - - Notes - ----- - The position is not zero based, but 1 based index. - If extraction is a string, :meth:`try_element_at` treats it as a column name, - while :meth:`element_at` treats it as a literal string. + A column that contains the minimum value computed. See Also -------- - :meth:`pyspark.sql.functions.get` - :meth:`pyspark.sql.functions.element_at` + :meth:`pyspark.sql.functions.max` + :meth:`pyspark.sql.functions.avg` + :meth:`pyspark.sql.functions.sum` Examples -------- - Example 1: Getting the first element of an array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit(1))).show() - +-----------------------+ - |try_element_at(data, 1)| - +-----------------------+ - | a| - +-----------------------+ - - Example 2: Getting the last element of an array using negative index + Example 1: Compute the minimum value of a numeric column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit(-1))).show() - +------------------------+ - |try_element_at(data, -1)| - +------------------------+ - | c| - +------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.range(10) + >>> df.select(sf.min(df.id)).show() + +-------+ + |min(id)| + +-------+ + | 0| + +-------+ - Example 3: Getting a value from a map using a key + Example 2: Compute the minimum value of a string column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit("a"))).show() - +-----------------------+ - |try_element_at(data, a)| - +-----------------------+ - | 1.0| - +-----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Alice",), ("Bob",), ("Charlie",)], ["name"]) + >>> df.select(sf.min("name")).show() + +---------+ + |min(name)| + +---------+ + | Alice| + +---------+ - Example 4: Getting a non-existing element from an array + Example 3: Compute the minimum value of a column with null values - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit(4))).show() - +-----------------------+ - |try_element_at(data, 4)| - +-----------------------+ - | NULL| - +-----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1,), (None,), (3,)], ["value"]) + >>> df.select(sf.min("value")).show() + +----------+ + |min(value)| + +----------+ + | 1| + +----------+ - Example 5: Getting a non-existing value from a map using a key + Example 4: Compute the minimum value of a column in a grouped DataFrame - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit("c"))).show() - +-----------------------+ - |try_element_at(data, c)| - +-----------------------+ - | NULL| - +-----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Alice", 1), ("Alice", 2), ("Bob", 3)], ["name", "value"]) + >>> df.groupBy("name").agg(sf.min("value")).show() + +-----+----------+ + | name|min(value)| + +-----+----------+ + |Alice| 1| + | Bob| 3| + +-----+----------+ - Example 6: Getting a value from a map using a column name as the key + Example 5: Compute the minimum value of a column in a DataFrame with multiple columns - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) - >>> df.select(sf.try_element_at(df.data, 'b')).show() - +-----------------------+ - |try_element_at(data, b)| - +-----------------------+ - | 1.0| - +-----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame( + ... [("Alice", 1, 100), ("Bob", 2, 200), ("Charlie", 3, 300)], + ... ["name", "value1", "value2"]) + >>> df.select(sf.min("value1"), sf.min("value2")).show() + +-----------+-----------+ + |min(value1)|min(value2)| + +-----------+-----------+ + | 1| 100| + +-----------+-----------+ """ - return _invoke_function_over_columns("try_element_at", col, extraction) + return _invoke_function_over_columns("min", col) @_try_remote_functions -def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column: +def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: """ - Array function: Returns the element of an array at the given (0-based) index. - If the index points outside of the array boundaries, then this function - returns NULL. + Returns the value(s) from the `col` parameter that are associated with the maximum value(s) + from the `ord` parameter. This function is often used to find the `col` parameter value + corresponding to the maximum `ord` parameter value within each group when used with groupBy(). - .. versionadded:: 3.4.0 + When `k` is specified, returns an array of up to `k` values associated with the top `k` + maximum values from `ord`. + + .. versionadded:: 3.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. versionchanged:: 4.2.0 + Added optional `k` parameter to return top-k values. + + Notes + ----- + The function is non-deterministic so the output order can be different for those + associated the same values of `col`. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of the column containing the array. - A column that evaluates to an array. - index : :class:`~pyspark.sql.Column` or str or int - Index to check for in the array. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + The column representing the values to be returned. This could be the column instance + or the column name as string. + A column of any type. + ord : :class:`~pyspark.sql.Column` or column name + The column that needs to be maximized. This could be the column instance + or the column name as string. + A column of any orderable type. + k : int, optional + If specified, returns an array of up to `k` values associated with the top `k` + maximum ordering values, sorted in descending order by the ordering column. + Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - Value at the given position. - Returns a column of the element type of the input array. - - Notes - ----- - The position is not 1-based, but 0-based index. - Supports Spark Connect. - - See Also - -------- - :meth:`pyspark.sql.functions.element_at` - :meth:`pyspark.sql.functions.try_element_at` + A column object representing the value from `col` that is associated with + the maximum value from `ord`. If `k` is specified, returns an array of values. Examples -------- - Example 1: Getting an element at a fixed position - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.get(df.data, 1)).show() - +------------+ - |get(data, 1)| - +------------+ - | b| - +------------+ + Example 1: Using `max_by` with groupBy - Example 2: Getting an element at a position outside the array boundaries + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.max_by("year", "earnings")).sort("course").show() + +------+----------------------+ + |course|max_by(year, earnings)| + +------+----------------------+ + | Java| 2013| + |dotNET| 2013| + +------+----------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.get(df.data, 3)).show() - +------------+ - |get(data, 3)| - +------------+ - | NULL| - +------------+ + Example 2: Using `max_by` with different data types - Example 3: Getting an element at a position specified by another column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), + ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.max_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|max_by(name, years_in_dept)| + +----------+---------------------------+ + | IT| Charlie| + | Marketing| Anna| + +----------+---------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) - >>> df.select(sf.get(df.data, df.index)).show() - +----------------+ - |get(data, index)| - +----------------+ - | c| - +----------------+ + Example 3: Using `max_by` where `ord` has multiple maximum values + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), + ... ("Finance", "George", 9), ("Consult", "Henry", 7)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.max_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|max_by(name, years_in_dept)| + +----------+---------------------------+ + | Consult| Henry| + | Finance| George| + +----------+---------------------------+ - Example 4: Getting an element at a position calculated from another column + Example 4: Using `max_by` with `k` to get top-k values - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) - >>> df.select(sf.get(df.data, df.index - 1)).show() - +----------------------+ - |get(data, (index - 1))| - +----------------------+ - | b| - +----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], + ... schema=("x", "y")) + >>> df.select(sf.max_by("x", "y", 2)).show() + +---------------+ + |max_by(x, y, 2)| + +---------------+ + | [b, d]| + +---------------+ + """ + if k is not None: + return _invoke_function_over_columns("max_by", col, ord, lit(k)) + return _invoke_function_over_columns("max_by", col, ord) - Example 5: Getting an element at a negative position - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], )], ['data']) - >>> df.select(sf.get(df.data, -1)).show() - +-------------+ - |get(data, -1)| - +-------------+ - | NULL| - +-------------+ +@_try_remote_functions +def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: """ - index = _enum_to_value(index) - index = lit(index) if isinstance(index, int) else index + Returns the value(s) from the `col` parameter that are associated with the minimum value(s) + from the `ord` parameter. This function is often used to find the `col` parameter value + corresponding to the minimum `ord` parameter value within each group when used with groupBy(). - return _invoke_function_over_columns("get", col, index) + When `k` is specified, returns an array of up to `k` values associated with the bottom `k` + minimum values from `ord`. + .. versionadded:: 3.3.0 -@_try_remote_functions -def array_prepend(col: "ColumnOrName", value: Any) -> Column: - """ - Array function: Returns an array containing the given element as - the first element and the rest of the elements from the original array. + .. versionchanged:: 3.4.0 + Supports Spark Connect. - .. versionadded:: 3.5.0 + .. versionchanged:: 4.2.0 + Added optional `k` parameter to return bottom-k values. + + Notes + ----- + The function is non-deterministic so the output order can be different for those + associated the same values of `col`. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing array. - A column that evaluates to an array. - value : - a literal value, or a :class:`~pyspark.sql.Column` expression. - A column of the same type as the array elements. + col : :class:`~pyspark.sql.Column` or column name + The column representing the values that will be returned. This could be the column instance + or the column name as string. + A column of any type. + ord : :class:`~pyspark.sql.Column` or column name + The column that needs to be minimized. This could be the column instance + or the column name as string. + A column of any orderable type. + k : int, optional + If specified, returns an array of up to `k` values associated with the bottom `k` + minimum ordering values, sorted in ascending order by the ordering column. + Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - an array with the given value prepended. - Returns a column that evaluates to an array. - - See Also - -------- - :meth:`pyspark.sql.functions.array_append` - :meth:`pyspark.sql.functions.array_insert` + Column object that represents the value from `col` associated with + the minimum value from `ord`. If `k` is specified, returns an array of values. Examples -------- - Example 1: Prepending a column value to an array column - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) - >>> df.select(sf.array_prepend(df.c1, df.c2)).show() - +---------------------+ - |array_prepend(c1, c2)| - +---------------------+ - | [c, b, a, c]| - +---------------------+ - - Example 2: Prepending a numeric value to an array column + Example 1: Using `min_by` with groupBy: - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_prepend(df.data, 4)).show() - +----------------------+ - |array_prepend(data, 4)| - +----------------------+ - | [4, 1, 2, 3]| - +----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.min_by("year", "earnings")).sort("course").show() + +------+----------------------+ + |course|min_by(year, earnings)| + +------+----------------------+ + | Java| 2012| + |dotNET| 2012| + +------+----------------------+ - Example 3: Prepending a null value to an array column + Example 2: Using `min_by` with different data types: - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_prepend(df.data, None)).show() - +-------------------------+ - |array_prepend(data, NULL)| - +-------------------------+ - | [NULL, 1, 2, 3]| - +-------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), + ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.min_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|min_by(name, years_in_dept)| + +----------+---------------------------+ + | IT| Bob| + | Marketing| David| + +----------+---------------------------+ - Example 4: Prepending a value to a NULL array column + Example 3: Using `min_by` where `ord` has multiple minimum values: - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([(None,)], schema=schema) - >>> df.select(sf.array_prepend(df.data, 4)).show() - +----------------------+ - |array_prepend(data, 4)| - +----------------------+ - | NULL| - +----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), + ... ("Finance", "George", 9), ("Consult", "Henry", 7)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.min_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|min_by(name, years_in_dept)| + +----------+---------------------------+ + | Consult| Eva| + | Finance| Frank| + +----------+---------------------------+ - Example 5: Prepending a value to an empty array + Example 4: Using `min_by` with `k` to get bottom-k values - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_prepend(df.data, 1)).show() - +----------------------+ - |array_prepend(data, 1)| - +----------------------+ - | [1]| - +----------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], + ... schema=("x", "y")) + >>> df.select(sf.min_by("x", "y", 2)).show() + +---------------+ + |min_by(x, y, 2)| + +---------------+ + | [a, c]| + +---------------+ """ - return _invoke_function_over_columns("array_prepend", col, lit(value)) + if k is not None: + return _invoke_function_over_columns("min_by", col, ord, lit(k)) + return _invoke_function_over_columns("min_by", col, ord) @_try_remote_functions -def array_remove(col: "ColumnOrName", element: Any) -> Column: +def count(col: "ColumnOrName") -> Column: """ - Array function: Remove all elements that equal to element from the given array. + Aggregate function: returns the number of items in a group. - .. versionadded:: 2.4.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing array. - A column that evaluates to an array. - element : - element or a :class:`~pyspark.sql.Column` expression to be removed from the array. - A column of the same type as the array elements. - - .. versionchanged:: 4.0.0 - `element` now also accepts a Column type. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. Returns ------- :class:`~pyspark.sql.Column` - A new column that is an array excluding the given value from the input column. - Returns a column that evaluates to an array. + column for computed results. See Also -------- - :meth:`pyspark.sql.functions.array_compact` + :meth:`pyspark.sql.functions.count_if` Examples -------- - Example 1: Removing a specific value from a simple array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],)], ['data']) - >>> df.select(sf.array_remove(df.data, 1)).show() - +---------------------+ - |array_remove(data, 1)| - +---------------------+ - | [2, 3]| - +---------------------+ - - Example 2: Removing a specific value from multiple arrays - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],), ([4, 5, 5, 4],)], ['data']) - >>> df.select(sf.array_remove(df.data, 5)).show() - +---------------------+ - |array_remove(data, 5)| - +---------------------+ - | [1, 2, 3, 1, 1]| - | [4, 4]| - +---------------------+ - - Example 3: Removing a value that does not exist in the array + Example 1: Count all rows in a DataFrame >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_remove(df.data, 4)).show() - +---------------------+ - |array_remove(data, 4)| - +---------------------+ - | [1, 2, 3]| - +---------------------+ + >>> df = spark.createDataFrame([(None,), ("a",), ("b",), ("c",)], schema=["alphabets"]) + >>> df.select(sf.count(sf.expr("*"))).show() + +--------+ + |count(1)| + +--------+ + | 4| + +--------+ - Example 4: Removing a value from an array with all identical values + Example 2: Count non-null values in a specific column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) - >>> df.select(sf.array_remove(df.data, 1)).show() - +---------------------+ - |array_remove(data, 1)| - +---------------------+ - | []| - +---------------------+ + >>> df.select(sf.count(df.alphabets)).show() + +----------------+ + |count(alphabets)| + +----------------+ + | 3| + +----------------+ - Example 5: Removing a value from an empty array + Example 3: Count all rows in a DataFrame with multiple columns >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema) - >>> df.select(sf.array_remove(df.data, 1)).show() - +---------------------+ - |array_remove(data, 1)| - +---------------------+ - | []| - +---------------------+ + >>> df = spark.createDataFrame( + ... [(1, "apple"), (2, "banana"), (3, None)], schema=["id", "fruit"]) + >>> df.select(sf.count(sf.expr("*"))).show() + +--------+ + |count(1)| + +--------+ + | 3| + +--------+ - Example 6: Removing a column's value from a simple array + Example 4: Count non-null values in multiple columns >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 1, 1], 1)], ['data', 'col']) - >>> df.select(sf.array_remove(df.data, df.col)).show() - +-----------------------+ - |array_remove(data, col)| - +-----------------------+ - | [2, 3]| - +-----------------------+ + >>> df.select(sf.count(df.id), sf.count(df.fruit)).show() + +---------+------------+ + |count(id)|count(fruit)| + +---------+------------+ + | 3| 2| + +---------+------------+ """ - return _invoke_function_over_columns("array_remove", col, lit(element)) + return _invoke_function_over_columns("count", col) @_try_remote_functions -def array_distinct(col: "ColumnOrName") -> Column: +def sum(col: "ColumnOrName") -> Column: """ - Array function: removes duplicate values from the array. + Aggregate function: returns the sum of all values in the expression. - .. versionadded:: 2.4.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` - A new column that is an array of unique values from the input column. - Returns a column that evaluates to an array. + the column for computed results. See Also -------- - :meth:`pyspark.sql.functions.array_except` - :meth:`pyspark.sql.functions.array_intersect` - :meth:`pyspark.sql.functions.array_union` + :meth:`pyspark.sql.functions.min` + :meth:`pyspark.sql.functions.max` + :meth:`pyspark.sql.functions.avg` Examples -------- - Example 1: Removing duplicate values from a simple array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 2],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1, 2, 3]| - +--------------------+ - - Example 2: Removing duplicate values from multiple arrays - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1, 2, 3]| - | [4, 5]| - +--------------------+ - - Example 3: Removing duplicate values from an array with all identical values + Example 1: Calculating the sum of values in a column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1]| - +--------------------+ + >>> df = spark.range(10) + >>> df.select(sf.sum(df["id"])).show() + +-------+ + |sum(id)| + +-------+ + | 45| + +-------+ - Example 4: Removing duplicate values from an array with no duplicate values + Example 2: Using a plus expression together to calculate the sum >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1, 2, 3]| - +--------------------+ + >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) + >>> df.select(sf.sum(sf.col("A") + sf.col("B"))).show() + +------------+ + |sum((A + B))| + +------------+ + | 10| + +------------+ - Example 5: Removing duplicate values from an empty array + Example 3: Calculating the summation of ages with None - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | []| - +--------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.sum("age")).show() + +--------+ + |sum(age)| + +--------+ + | 6| + +--------+ """ - return _invoke_function_over_columns("array_distinct", col) + return _invoke_function_over_columns("sum", col) @_try_remote_functions -def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: Any) -> Column: +def avg(col: "ColumnOrName") -> Column: """ - Array function: Inserts an item into a given array at a specified array index. - Array indices start at 1, or start from the end if index is negative. - Index above array size appends the array, or prepends the array if index is negative, - with 'null' elements. + Aggregate function: returns the average of the values in a group. - .. versionadded:: 3.4.0 + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - arr : :class:`~pyspark.sql.Column` or str - name of column containing an array. - A column that evaluates to an array. - pos : :class:`~pyspark.sql.Column` or str or int - name of integral type column indicating position of insertion - (starting at index 1, negative position is a start from the back of the array). - A column that evaluates to an integer. - value : - a literal value, or a :class:`~pyspark.sql.Column` expression. - A column of the same type as the array elements. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` - an array of values, including the new specified value - Returns a column that evaluates to an array. - - Notes - ----- - Supports Spark Connect. + the column for computed results. See Also -------- - :meth:`pyspark.sql.functions.array_append` - :meth:`pyspark.sql.functions.array_prepend` + :meth:`pyspark.sql.functions.min` + :meth:`pyspark.sql.functions.max` + :meth:`pyspark.sql.functions.sum` Examples -------- - Example 1: Inserting a value at a specific position + Example 1: Calculating the average age - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, 2, 'd')).show() - +------------------------+ - |array_insert(data, 2, d)| - +------------------------+ - | [a, d, b, c]| - +------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) + >>> df.select(sf.avg("age")).show() + +--------+ + |avg(age)| + +--------+ + | 8.5| + +--------+ - Example 2: Inserting a value at a negative position + Example 2: Calculating the average age with None - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, -2, 'd')).show() - +-------------------------+ - |array_insert(data, -2, d)| - +-------------------------+ - | [a, b, d, c]| - +-------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.avg("age")).show() + +--------+ + |avg(age)| + +--------+ + | 3.0| + +--------+ + """ + return _invoke_function_over_columns("avg", col) - Example 3: Inserting a value at a position greater than the array size - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, 5, 'e')).show() - +------------------------+ - |array_insert(data, 5, e)| - +------------------------+ - | [a, b, c, NULL, e]| - +------------------------+ +@_try_remote_functions +def mean(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns the average of the values in a group. + An alias of :func:`avg`. - Example 4: Inserting a NULL value + .. versionadded:: 1.4.0 - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, 2, sf.lit(None))).show() - +---------------------------+ - |array_insert(data, 2, NULL)| - +---------------------------+ - | [a, NULL, b, c]| - +---------------------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Example 5: Inserting a value into a NULL array + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric or interval. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) - >>> df = spark.createDataFrame([(None,)], schema=schema) - >>> df.select(sf.array_insert(df.data, 1, 5)).show() - +------------------------+ - |array_insert(data, 1, 5)| - +------------------------+ - | NULL| - +------------------------+ - """ - pos = _enum_to_value(pos) - pos = lit(pos) if isinstance(pos, int) else pos + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. - return _invoke_function_over_columns("array_insert", arr, pos, lit(value)) + Examples + -------- + Example 1: Calculating the average age + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) + >>> df.select(sf.mean("age")).show() + +--------+ + |avg(age)| + +--------+ + | 8.5| + +--------+ + Example 2: Calculating the average age with None -@_try_remote_functions -def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.mean("age")).show() + +--------+ + |avg(age)| + +--------+ + | 3.0| + +--------+ """ - Array function: returns a new array containing the intersection of elements in col1 and col2, - without duplicates. + return _invoke_function_over_columns("mean", col) - .. versionadded:: 2.4.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def median(col: "ColumnOrName") -> Column: + """ + Returns the median of the values in a group. + + .. versionadded:: 3.4.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - Name of column containing the first array. - A column that evaluates to an array. - col2 : :class:`~pyspark.sql.Column` or str - Name of column containing the second array. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric, interval, or time. Returns ------- :class:`~pyspark.sql.Column` - A new array containing the intersection of elements in col1 and col2. - Returns a column that evaluates to an array. + the median of the values in a group. Notes ----- - This function does not preserve the order of the elements in the input arrays. + Supports Spark Connect. + + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.approx_percentile` + :meth:`pyspark.sql.functions.percentile_approx` See Also -------- - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.array_except` - :meth:`pyspark.sql.functions.array_union` + :meth:`pyspark.sql.functions.approx_percentile` + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.percentile_approx` Examples -------- - Example 1: Basic usage - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) - >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() - +-----------------------------------------+ - |sort_array(array_intersect(c1, c2), true)| - +-----------------------------------------+ - | [a, c]| - +-----------------------------------------+ - - Example 2: Intersection with no common elements - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) - >>> df.select(sf.array_intersect(df.c1, df.c2)).show() - +-----------------------+ - |array_intersect(c1, c2)| - +-----------------------+ - | []| - +-----------------------+ - - Example 3: Intersection with all common elements + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("Java", 2012, 22000), ("dotNET", 2012, 10000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.median("earnings")).show() + +------+----------------+ + |course|median(earnings)| + +------+----------------+ + | Java| 22000.0| + |dotNET| 10000.0| + +------+----------------+ + """ + return _invoke_function_over_columns("median", col) - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) - >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() - +-----------------------------------------+ - |sort_array(array_intersect(c1, c2), true)| - +-----------------------------------------+ - | [a, b, c]| - +-----------------------------------------+ - Example 4: Intersection with null values +@_try_remote_functions +def sumDistinct(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns the sum of distinct values in the expression. - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) - >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() - +-----------------------------------------+ - |sort_array(array_intersect(c1, c2), true)| - +-----------------------------------------+ - | [NULL, a]| - +-----------------------------------------+ + .. versionadded:: 1.3.0 - Example 5: Intersection with empty arrays + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> data = [Row(c1=[], c2=["a", "b", "c"])] - >>> schema = StructType([ - ... StructField("c1", ArrayType(StringType()), True), - ... StructField("c2", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.array_intersect(df.c1, df.c2)).show() - +-----------------------+ - |array_intersect(c1, c2)| - +-----------------------+ - | []| - +-----------------------+ + .. deprecated:: 3.2.0 + Use :func:`sum_distinct` instead. """ - return _invoke_function_over_columns("array_intersect", col1, col2) + warnings.warn("Deprecated in 3.2, use sum_distinct instead.", FutureWarning) + return sum_distinct(col) @_try_remote_functions -def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def sum_distinct(col: "ColumnOrName") -> Column: """ - Array function: returns a new array containing the union of elements in col1 and col2, - without duplicates. + Aggregate function: returns the sum of distinct values in the expression. - .. versionadded:: 2.4.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - Name of column containing the first array. - A column that evaluates to an array. - col2 : :class:`~pyspark.sql.Column` or str - Name of column containing the second array. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. Returns ------- :class:`~pyspark.sql.Column` - A new array containing the union of elements in col1 and col2. - Returns a column that evaluates to an array. - - Notes - ----- - This function does not preserve the order of the elements in the input arrays. - - See Also - -------- - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.array_except` - :meth:`pyspark.sql.functions.array_intersect` + the column for computed results. Examples -------- - Example 1: Basic usage - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c, d, f]| - +-------------------------------------+ - - Example 2: Union with no common elements + Example 1: Using sum_distinct function on a column with all distinct values - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c, d, e, f]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,)], ["numbers"]) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | 10| + +---------------------+ - Example 3: Union with all common elements + Example 2: Using sum_distinct function on a column with no distinct values - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,), (1,), (1,), (1,)], ["numbers"]) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | 1| + +---------------------+ - Example 4: Union with null values + Example 3: Using sum_distinct function on a column with null and duplicate values - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [NULL, a, b, c]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(None,), (1,), (1,), (2,)], ["numbers"]) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | 3| + +---------------------+ - Example 5: Union with empty arrays + Example 4: Using sum_distinct function on a column with all None values - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> data = [Row(c1=[], c2=["a", "b", "c"])] - >>> schema = StructType([ - ... StructField("c1", ArrayType(StringType()), True), - ... StructField("c2", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, IntegerType + >>> schema = StructType([StructField("numbers", IntegerType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | NULL| + +---------------------+ """ - return _invoke_function_over_columns("array_union", col1, col2) + return _invoke_function_over_columns("sum_distinct", col) @_try_remote_functions -def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None) -> Column: """ - Array function: returns a new array containing the elements present in col1 but not in col2, - without duplicates. - - .. versionadded:: 2.4.0 + Aggregate function: returns the concatenation of non-null input values, + separated by the delimiter. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - Name of column containing the first array. - A column that evaluates to an array. - col2 : :class:`~pyspark.sql.Column` or str - Name of column containing the second array. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a string or binary. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new array containing the elements present in col1 but not in col2. - Returns a column that evaluates to an array. - - Notes - ----- - This function does not preserve the order of the elements in the input arrays. - - See Also - -------- - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.array_intersect` - :meth:`pyspark.sql.functions.array_union` + the column for computed results. Examples -------- - Example 1: Basic usage - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) - >>> df.select(sf.array_except(df.c1, df.c2)).show() - +--------------------+ - |array_except(c1, c2)| - +--------------------+ - | [b]| - +--------------------+ - - Example 2: Except with no common elements + Example 1: Using listagg function - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) - >>> df.select(sf.sort_array(sf.array_except(df.c1, df.c2))).show() - +--------------------------------------+ - |sort_array(array_except(c1, c2), true)| - +--------------------------------------+ - | [a, b, c]| - +--------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.listagg('strings')).show() + +----------------------+ + |listagg(strings, NULL)| + +----------------------+ + | abc| + +----------------------+ - Example 3: Except with all common elements + Example 2: Using listagg function with a delimiter - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) - >>> df.select(sf.array_except(df.c1, df.c2)).show() + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.listagg('strings', ', ')).show() +--------------------+ - |array_except(c1, c2)| + |listagg(strings, , )| +--------------------+ - | []| + | a, b, c| +--------------------+ - Example 4: Except with null values + Example 3: Using listagg function with a binary column and delimiter - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) - >>> df.select(sf.array_except(df.c1, df.c2)).show() - +--------------------+ - |array_except(c1, c2)| - +--------------------+ - | [b]| - +--------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) + >>> df.select(sf.listagg('bytes', b'\x42')).show() + +---------------------+ + |listagg(bytes, X'42')| + +---------------------+ + | [01 42 02 42 03]| + +---------------------+ - Example 5: Except with empty arrays + Example 4: Using listagg function on a column with all None values - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> data = [Row(c1=[], c2=["a", "b", "c"])] - >>> schema = StructType([ - ... StructField("c1", ArrayType(StringType()), True), - ... StructField("c2", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.array_except(df.c1, df.c2)).show() - +--------------------+ - |array_except(c1, c2)| - +--------------------+ - | []| - +--------------------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.listagg('strings')).show() + +----------------------+ + |listagg(strings, NULL)| + +----------------------+ + | NULL| + +----------------------+ """ - return _invoke_function_over_columns("array_except", col1, col2) + if delimiter is None: + return _invoke_function_over_columns("listagg", col) + else: + return _invoke_function_over_columns("listagg", col, lit(delimiter)) @_try_remote_functions -def array_compact(col: "ColumnOrName") -> Column: +def listagg_distinct( + col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None +) -> Column: """ - Array function: removes null values from the array. + Aggregate function: returns the concatenation of distinct non-null input values, + separated by the delimiter. - .. versionadded:: 3.4.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. Returns ------- :class:`~pyspark.sql.Column` - A new column that is an array excluding the null values from the input column. - Returns a column that evaluates to an array. - - Notes - ----- - Supports Spark Connect. - - See Also - -------- - :meth:`pyspark.sql.functions.array_remove` + the column for computed results. Examples -------- - Example 1: Removing null values from a simple array + Example 1: Using listagg_distinct function >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, None, 2, 3],)], ['data']) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | [1, 2, 3]| - +-------------------+ + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.listagg_distinct('strings')).show() + +-------------------------------+ + |listagg(DISTINCT strings, NULL)| + +-------------------------------+ + | abc| + +-------------------------------+ - Example 2: Removing null values from multiple arrays + Example 2: Using listagg_distinct function with a delimiter >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, None, 2, 3],), ([4, 5, None, 4],)], ['data']) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | [1, 2, 3]| - | [4, 5, 4]| - +-------------------+ + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.listagg_distinct('strings', ', ')).show() + +-----------------------------+ + |listagg(DISTINCT strings, , )| + +-----------------------------+ + | a, b, c| + +-----------------------------+ - Example 3: Removing null values from an array with all null values + Example 3: Using listagg_distinct function with a binary column and delimiter >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([ - ... StructField("data", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame([([None, None, None],)], schema) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | []| - +-------------------+ + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], + ... ['bytes']) + >>> df.select(sf.listagg_distinct('bytes', b'\x42')).show() + +------------------------------+ + |listagg(DISTINCT bytes, X'42')| + +------------------------------+ + | [01 42 02 42 03]| + +------------------------------+ - Example 4: Removing null values from an array with no null values + Example 4: Using listagg_distinct function on a column with all None values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | [1, 2, 3]| - +-------------------+ - - Example 5: Removing null values from an empty array - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([ - ... StructField("data", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | []| - +-------------------+ - """ - return _invoke_function_over_columns("array_compact", col) + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.listagg_distinct('strings')).show() + +-------------------------------+ + |listagg(DISTINCT strings, NULL)| + +-------------------------------+ + | NULL| + +-------------------------------+ + """ + if delimiter is None: + return _invoke_function_over_columns("listagg_distinct", col) + else: + return _invoke_function_over_columns("listagg_distinct", col, lit(delimiter)) @_try_remote_functions -def array_append(col: "ColumnOrName", value: Any) -> Column: +def string_agg( + col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None +) -> Column: """ - Array function: returns a new array column by appending `value` to the existing array `col`. + Aggregate function: returns the concatenation of non-null input values, + separated by the delimiter. - .. versionadded:: 3.4.0 + An alias of :func:`listagg`. + + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column containing the array. - A column that evaluates to an array. - value : - A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. - A column of the same type as the array elements. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a string or binary. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new array column with `value` appended to the original array. - Returns a column that evaluates to an array. - - Notes - ----- - Supports Spark Connect. - - See Also - -------- - :meth:`pyspark.sql.functions.array_insert` - :meth:`pyspark.sql.functions.array_prepend` + the column for computed results. Examples -------- - Example 1: Appending a column value to an array column + Example 1: Using string_agg function - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) - >>> df.select(sf.array_append(df.c1, df.c2)).show() - +--------------------+ - |array_append(c1, c2)| - +--------------------+ - | [b, a, c, c]| - +--------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.string_agg('strings')).show() + +-------------------------+ + |string_agg(strings, NULL)| + +-------------------------+ + | abc| + +-------------------------+ - Example 2: Appending a numeric value to an array column + Example 2: Using string_agg function with a delimiter >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_append(df.data, 4)).show() - +---------------------+ - |array_append(data, 4)| - +---------------------+ - | [1, 2, 3, 4]| - +---------------------+ + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.string_agg('strings', ', ')).show() + +-----------------------+ + |string_agg(strings, , )| + +-----------------------+ + | a, b, c| + +-----------------------+ - Example 3: Appending a null value to an array column + Example 3: Using string_agg function with a binary column and delimiter >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_append(df.data, None)).show() + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) + >>> df.select(sf.string_agg('bytes', b'\x42')).show() +------------------------+ - |array_append(data, NULL)| + |string_agg(bytes, X'42')| +------------------------+ - | [1, 2, 3, NULL]| + | [01 42 02 42 03]| +------------------------+ - Example 4: Appending a value to a NULL array column - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([(None,)], schema=schema) - >>> df.select(sf.array_append(df.data, 4)).show() - +---------------------+ - |array_append(data, 4)| - +---------------------+ - | NULL| - +---------------------+ - - Example 5: Appending a value to an empty array + Example 4: Using string_agg function on a column with all None values >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_append(df.data, 1)).show() - +---------------------+ - |array_append(data, 1)| - +---------------------+ - | [1]| - +---------------------+ + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.string_agg('strings')).show() + +-------------------------+ + |string_agg(strings, NULL)| + +-------------------------+ + | NULL| + +-------------------------+ """ - return _invoke_function_over_columns("array_append", col, lit(value)) + if delimiter is None: + return _invoke_function_over_columns("string_agg", col) + else: + return _invoke_function_over_columns("string_agg", col, lit(delimiter)) @_try_remote_functions -def explode(col: "ColumnOrName") -> Column: +def string_agg_distinct( + col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None +) -> Column: """ - Returns a new row for each element in the given array or map. - Uses the default column name `col` for elements in the array and - `key` and `value` for elements in the map unless specified otherwise. + Aggregate function: returns the concatenation of distinct non-null input values, + separated by the delimiter. - .. versionadded:: 1.4.0 + An alias of :func:`listagg_distinct`. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Target column to work on. - A column that evaluates to an array or map. + target column to compute on. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. Returns ------- :class:`~pyspark.sql.Column` - One row per array item or map key value. - Returns a column of the element type of the input array, or the key and value - columns of the input map. - - See Also - -------- - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` - - Notes - ----- - Only one explode is allowed per SELECT clause. + the column for computed results. Examples -------- - Example 1: Exploding an array column + Example 1: Using string_agg_distinct function >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.show() - +---+---------------+ - | i| a| - +---+---------------+ - | 1|[1, 2, 3, NULL]| - | 2| []| - | 3| NULL| - +---+---------------+ - - >>> df.select('*', sf.explode('a')).show() - +---+---------------+----+ - | i| a| col| - +---+---------------+----+ - | 1|[1, 2, 3, NULL]| 1| - | 1|[1, 2, 3, NULL]| 2| - | 1|[1, 2, 3, NULL]| 3| - | 1|[1, 2, 3, NULL]|NULL| - +---+---------------+----+ + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.string_agg_distinct('strings')).show() + +----------------------------------+ + |string_agg(DISTINCT strings, NULL)| + +----------------------------------+ + | abc| + +----------------------------------+ - Example 2: Exploding a map column + Example 2: Using string_agg_distinct function with a delimiter >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.show(truncate=False) - +---+---------------------------+ - |i |m | - +---+---------------------------+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| - |2 |{} | - |3 |NULL | - +---+---------------------------+ - - >>> df.select('*', sf.explode('m')).show(truncate=False) - +---+---------------------------+---+-----+ - |i |m |key|value| - +---+---------------------------+---+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | - +---+---------------------------+---+-----+ + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.string_agg_distinct('strings', ', ')).show() + +--------------------------------+ + |string_agg(DISTINCT strings, , )| + +--------------------------------+ + | a, b, c| + +--------------------------------+ - Example 3: Exploding multiple array columns + Example 3: Using string_agg_distinct function with a binary column and delimiter - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(1,2) AS a1, ARRAY(3,4,5) AS a2') - >>> df.select( - ... '*', sf.explode('a1').alias('v1') - ... ).select('*', sf.explode('a2').alias('v2')).show() - +------+---------+---+---+ - | a1| a2| v1| v2| - +------+---------+---+---+ - |[1, 2]|[3, 4, 5]| 1| 3| - |[1, 2]|[3, 4, 5]| 1| 4| - |[1, 2]|[3, 4, 5]| 1| 5| - |[1, 2]|[3, 4, 5]| 2| 3| - |[1, 2]|[3, 4, 5]| 2| 4| - |[1, 2]|[3, 4, 5]| 2| 5| - +------+---------+---+---+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], + ... ['bytes']) + >>> df.select(sf.string_agg_distinct('bytes', b'\x42')).show() + +---------------------------------+ + |string_agg(DISTINCT bytes, X'42')| + +---------------------------------+ + | [01 42 02 42 03]| + +---------------------------------+ - Example 4: Exploding an array of struct column + Example 4: Using string_agg_distinct function on a column with all None values - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select(sf.explode('a').alias("s")).select("s.*").show() - +---+---+ - | a| b| - +---+---+ - | 1| 2| - | 3| 4| - +---+---+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.string_agg_distinct('strings')).show() + +----------------------------------+ + |string_agg(DISTINCT strings, NULL)| + +----------------------------------+ + | NULL| + +----------------------------------+ """ - return _invoke_function_over_columns("explode", col) + if delimiter is None: + return _invoke_function_over_columns("string_agg_distinct", col) + else: + return _invoke_function_over_columns("string_agg_distinct", col, lit(delimiter)) @_try_remote_functions -def posexplode(col: "ColumnOrName") -> Column: +def product(col: "ColumnOrName") -> Column: """ - Returns a new row for each element with position in the given array or map. - Uses the default column name `pos` for position, and `col` for elements in the - array and `key` and `value` for elements in the map unless specified otherwise. + Aggregate function: returns the product of the values in a group. - .. versionadded:: 2.1.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -22190,269 +21664,121 @@ def posexplode(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to work on. + column containing values to be multiplied together Returns ------- - :class:`~pyspark.sql.Column` - one row per array item or map key value including positions as a separate column. - - See Also - -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` + :class:`~pyspark.sql.Column` or column name + the column for computed results. Examples -------- - Example 1: Exploding an array column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.show() - +---+---------------+ - | i| a| - +---+---------------+ - | 1|[1, 2, 3, NULL]| - | 2| []| - | 3| NULL| - +---+---------------+ - - >>> df.select('*', sf.posexplode('a')).show() - +---+---------------+---+----+ - | i| a|pos| col| - +---+---------------+---+----+ - | 1|[1, 2, 3, NULL]| 0| 1| - | 1|[1, 2, 3, NULL]| 1| 2| - | 1|[1, 2, 3, NULL]| 2| 3| - | 1|[1, 2, 3, NULL]| 3|NULL| - +---+---------------+---+----+ - - Example 2: Exploding a map column - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.show(truncate=False) - +---+---------------------------+ - |i |m | - +---+---------------------------+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| - |2 |{} | - |3 |NULL | - +---+---------------------------+ - - >>> df.select('*', sf.posexplode('m')).show(truncate=False) - +---+---------------------------+---+---+-----+ - |i |m |pos|key|value| - +---+---------------------------+---+---+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | - +---+---------------------------+---+---+-----+ + >>> df = spark.sql("SELECT id % 3 AS mod3, id AS value FROM RANGE(10)") + >>> df.groupBy('mod3').agg(sf.product('value')).orderBy('mod3').show() + +----+--------------+ + |mod3|product(value)| + +----+--------------+ + | 0| 0.0| + | 1| 28.0| + | 2| 80.0| + +----+--------------+ """ - return _invoke_function_over_columns("posexplode", col) + return _invoke_function_over_columns("product", col) @_try_remote_functions -def inline(col: "ColumnOrName") -> Column: +def stddev(col: "ColumnOrName") -> Column: """ - Explodes an array of structs into a table. + Aggregate function: alias for stddev_samp. - This function takes an input column containing an array of structs and returns a - new column where each struct in the array is exploded into a separate row. + .. versionadded:: 1.6.0 - .. versionadded:: 3.4.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Input column of values to explode. + target column to compute on. + A column that evaluates to a numeric. + + See Also + -------- + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.skewness` + :meth:`pyspark.sql.functions.kurtosis` Returns ------- :class:`~pyspark.sql.Column` - Generator expression with the inline exploded result. - - See Also - -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline_outer` + standard deviation of given column. + Returns a column that evaluates to a double. Examples -------- - Example 1: Using inline with a single struct array column - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select('*', sf.inline(df.a)).show() - +----------------+---+---+ - | a| a| b| - +----------------+---+---+ - |[{1, 2}, {3, 4}]| 1| 2| - |[{1, 2}, {3, 4}]| 3| 4| - +----------------+---+---+ - - Example 2: Using inline with a column name - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select('*', sf.inline('a')).show() - +----------------+---+---+ - | a| a| b| - +----------------+---+---+ - |[{1, 2}, {3, 4}]| 1| 2| - |[{1, 2}, {3, 4}]| 3| 4| - +----------------+---+---+ - - Example 3: Using inline with an alias - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select('*', sf.inline('a').alias("c1", "c2")).show() - +----------------+---+---+ - | a| c1| c2| - +----------------+---+---+ - |[{1, 2}, {3, 4}]| 1| 2| - |[{1, 2}, {3, 4}]| 3| 4| - +----------------+---+---+ - - Example 4: Using inline with multiple struct array columns - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a1, ARRAY(NAMED_STRUCT("c",5,"d",6), NAMED_STRUCT("c",7,"d",8)) AS a2') - >>> df.select( - ... '*', sf.inline('a1') - ... ).select('*', sf.inline('a2')).show() - +----------------+----------------+---+---+---+---+ - | a1| a2| a| b| c| d| - +----------------+----------------+---+---+---+---+ - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 5| 6| - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 7| 8| - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 5| 6| - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 7| 8| - +----------------+----------------+---+---+---+---+ - - Example 5: Using inline with a nested struct array column - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT NAMED_STRUCT("a",1,"b",2,"c",ARRAY(NAMED_STRUCT("c",3,"d",4), NAMED_STRUCT("c",5,"d",6))) AS s') - >>> df.select('*', sf.inline('s.c')).show(truncate=False) - +------------------------+---+---+ - |s |c |d | - +------------------------+---+---+ - |{1, 2, [{3, 4}, {5, 6}]}|3 |4 | - |{1, 2, [{3, 4}, {5, 6}]}|5 |6 | - +------------------------+---+---+ - - Example 6: Using inline with a column containing: array continaing null, empty array and null - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') - >>> df.show(truncate=False) - +---+----------------------+ - |i |s | - +---+----------------------+ - |1 |[{1, 2}, NULL, {3, 4}]| - |2 |[] | - |3 |NULL | - +---+----------------------+ - - >>> df.select('*', sf.inline('s')).show(truncate=False) - +---+----------------------+----+----+ - |i |s |a |b | - +---+----------------------+----+----+ - |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | - |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| - |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | - +---+----------------------+----+----+ + >>> spark.range(6).select(sf.stddev("id")).show() + +------------------+ + | stddev(id)| + +------------------+ + |1.8708286933869...| + +------------------+ """ - return _invoke_function_over_columns("inline", col) + return _invoke_function_over_columns("stddev", col) @_try_remote_functions -def explode_outer(col: "ColumnOrName") -> Column: +def std(col: "ColumnOrName") -> Column: """ - Returns a new row for each element in the given array or map. - Unlike explode, if the array/map is null or empty then null is produced. - Uses the default column name `col` for elements in the array and - `key` and `value` for elements in the map unless specified otherwise. - - .. versionadded:: 2.3.0 + Aggregate function: alias for stddev_samp. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to an array or map. + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - one row per array item or map key value. - Returns a column of the element type of the input array, or the key and value - columns of the input map. + standard deviation of given column. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.skewness` + :meth:`pyspark.sql.functions.kurtosis` Examples -------- - Example 1: Using an array column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.select('*', sf.explode_outer('a')).show() - +---+---------------+----+ - | i| a| col| - +---+---------------+----+ - | 1|[1, 2, 3, NULL]| 1| - | 1|[1, 2, 3, NULL]| 2| - | 1|[1, 2, 3, NULL]| 3| - | 1|[1, 2, 3, NULL]|NULL| - | 2| []|NULL| - | 3| NULL|NULL| - +---+---------------+----+ - - Example 2: Using a map column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.select('*', sf.explode_outer('m')).show(truncate=False) - +---+---------------------------+----+-----+ - |i |m |key |value| - +---+---------------------------+----+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | - |2 |{} |NULL|NULL | - |3 |NULL |NULL|NULL | - +---+---------------------------+----+-----+ + >>> import pyspark.sql.functions as sf + >>> spark.range(6).select(sf.std("id")).show() + +------------------+ + | std(id)| + +------------------+ + |1.8708286933869...| + +------------------+ """ - return _invoke_function_over_columns("explode_outer", col) + return _invoke_function_over_columns("std", col) @_try_remote_functions -def posexplode_outer(col: "ColumnOrName") -> Column: +def stddev_samp(col: "ColumnOrName") -> Column: """ - Returns a new row for each element with position in the given array or map. - Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced. - Uses the default column name `pos` for position, and `col` for elements in the - array and `key` and `value` for elements in the map unless specified otherwise. + Aggregate function: returns the unbiased sample standard deviation of + the expression in a group. - .. versionadded:: 2.3.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -22460,118 +21786,82 @@ def posexplode_outer(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to work on. + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - one row per array item or map key value including positions as a separate column. + standard deviation of given column. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.var_samp` Examples -------- - Example 1: Using an array column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.select('*', sf.posexplode_outer('a')).show() - +---+---------------+----+----+ - | i| a| pos| col| - +---+---------------+----+----+ - | 1|[1, 2, 3, NULL]| 0| 1| - | 1|[1, 2, 3, NULL]| 1| 2| - | 1|[1, 2, 3, NULL]| 2| 3| - | 1|[1, 2, 3, NULL]| 3|NULL| - | 2| []|NULL|NULL| - | 3| NULL|NULL|NULL| - +---+---------------+----+----+ - - Example 2: Using a map column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.select('*', sf.posexplode_outer('m')).show(truncate=False) - +---+---------------------------+----+----+-----+ - |i |m |pos |key |value| - +---+---------------------------+----+----+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | - |2 |{} |NULL|NULL|NULL | - |3 |NULL |NULL|NULL|NULL | - +---+---------------------------+----+----+-----+ + >>> import pyspark.sql.functions as sf + >>> spark.range(6).select(sf.stddev_samp("id")).show() + +------------------+ + | stddev_samp(id)| + +------------------+ + |1.8708286933869...| + +------------------+ """ - return _invoke_function_over_columns("posexplode_outer", col) + return _invoke_function_over_columns("stddev_samp", col) @_try_remote_functions -def inline_outer(col: "ColumnOrName") -> Column: +def stddev_pop(col: "ColumnOrName") -> Column: """ - Explodes an array of structs into a table. - Unlike inline, if the array is null or empty then null is produced for each nested column. + Aggregate function: returns population standard deviation of + the expression in a group. - .. versionadded:: 3.4.0 + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - input column of values to explode. + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - generator expression with the inline exploded result. + standard deviation of given column. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - - Notes - ----- - Supports Spark Connect. + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.var_pop` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') - >>> df.printSchema() - root - |-- i: integer (nullable = false) - |-- s: array (nullable = true) - | |-- element: struct (containsNull = true) - | | |-- a: integer (nullable = false) - | | |-- b: integer (nullable = false) - - >>> df.select('*', sf.inline_outer('s')).show(truncate=False) - +---+----------------------+----+----+ - |i |s |a |b | - +---+----------------------+----+----+ - |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | - |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| - |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | - |2 |[] |NULL|NULL| - |3 |NULL |NULL|NULL| - +---+----------------------+----+----+ + >>> import pyspark.sql.functions as sf + >>> spark.range(6).select(sf.stddev_pop("id")).show() + +-----------------+ + | stddev_pop(id)| + +-----------------+ + |1.707825127659...| + +-----------------+ """ - return _invoke_function_over_columns("inline_outer", col) + return _invoke_function_over_columns("stddev_pop", col) @_try_remote_functions -def get_json_object(col: "ColumnOrName", path: str) -> Column: +def variance(col: "ColumnOrName") -> Column: """ - Extracts json object from a json string based on json `path` specified, and returns json string - of the extracted json object. It will return null if the input json string is invalid. + Aggregate function: alias for var_samp .. versionadded:: 1.6.0 @@ -22580,73 +21870,43 @@ def get_json_object(col: "ColumnOrName", path: str) -> Column: Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - string column in json format. - A column that evaluates to a string. - path : str - path to the json object to extract. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - string representation of given JSON object value. - Returns a column that evaluates to a string. + variance of given column. - Examples + See Also -------- - Example 1: Extract a json object from json string - - >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] - >>> df = spark.createDataFrame(data, ("key", "jstring")) - >>> df.select(df.key, - ... get_json_object(df.jstring, '$.f1').alias("c0"), - ... get_json_object(df.jstring, '$.f2').alias("c1") - ... ).show() - +---+-------+------+ - |key| c0| c1| - +---+-------+------+ - | 1| value1|value2| - | 2|value12| NULL| - +---+-------+------+ - - Example 2: Extract a json object from json array - - >>> data = [ - ... ("1", '''[{"f1": "value1"},{"f1": "value2"}]'''), - ... ("2", '''[{"f1": "value12"},{"f2": "value13"}]''') - ... ] - >>> df = spark.createDataFrame(data, ("key", "jarray")) - >>> df.select(df.key, - ... get_json_object(df.jarray, '$[0].f1').alias("c0"), - ... get_json_object(df.jarray, '$[1].f2').alias("c1") - ... ).show() - +---+-------+-------+ - |key| c0| c1| - +---+-------+-------+ - | 1| value1| NULL| - | 2|value12|value13| - +---+-------+-------+ + :meth:`pyspark.sql.functions.var_pop` + :meth:`pyspark.sql.functions.var_samp` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.skewness` + :meth:`pyspark.sql.functions.kurtosis` + :meth:`pyspark.sql.functions.std` - >>> df.select(df.key, - ... get_json_object(df.jarray, '$[*].f1').alias("c0"), - ... get_json_object(df.jarray, '$[*].f2').alias("c1") - ... ).show() - +---+-------------------+---------+ - |key| c0| c1| - +---+-------------------+---------+ - | 1|["value1","value2"]| NULL| - | 2| "value12"|"value13"| - +---+-------------------+---------+ + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.range(6) + >>> df.select(sf.variance(df.id)).show() + +------------+ + |variance(id)| + +------------+ + | 3.5| + +------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("get_json_object", _to_java_column(col), _enum_to_value(path)) + return _invoke_function_over_columns("variance", col) @_try_remote_functions -def json_tuple(col: "ColumnOrName", *fields: str) -> Column: - """Creates a new row for a json column according to the given field names. +def var_samp(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns the unbiased sample variance of + the values in a group. .. versionadded:: 1.6.0 @@ -22655,10817 +21915,11820 @@ def json_tuple(col: "ColumnOrName", *fields: str) -> Column: Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - string column in json format - A column that evaluates to a string. - fields : str - a field or fields to extract - Each a column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a new row for each given field value from json object - Returns a column that evaluates to a string. + variance of given column. + + See Also + -------- + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.var_pop` + :meth:`pyspark.sql.functions.stddev_samp` Examples -------- - >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] - >>> df = spark.createDataFrame(data, ("key", "jstring")) - >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect() - [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)] + >>> from pyspark.sql import functions as sf + >>> df = spark.range(6) + >>> df.select(sf.var_samp(df.id)).show() + +------------+ + |var_samp(id)| + +------------+ + | 3.5| + +------------+ """ - from pyspark.sql.classic.column import _to_java_column, _to_seq - - if len(fields) == 0: - raise PySparkValueError( - errorClass="CANNOT_BE_EMPTY", - messageParameters={"item": "field"}, - ) - sc = _get_active_spark_context() - return _invoke_function("json_tuple", _to_java_column(col), _to_seq(sc, fields)) + return _invoke_function_over_columns("var_samp", col) @_try_remote_functions -def from_json( - col: "ColumnOrName", - schema: Union[ArrayType, StructType, MapType, Column, str], - options: Optional[Mapping[str, str]] = None, -) -> Column: +def var_pop(col: "ColumnOrName") -> Column: """ - Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType` - as keys type, :class:`StructType` or :class:`ArrayType` with - the specified schema. Returns `null`, in the case of an unparsable string. + Aggregate function: returns the population variance of the values in a group. - .. versionadded:: 2.1.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a column or column name in JSON format - schema : :class:`StructType`, :class:`ArrayType`, :class:`MapType`, or str - a StructType, ArrayType of StructType, MapType, or Python string literal with a DDL-formatted string - A column that evaluates to a string, or a DDL-formatted type string, or a DataType. - to use when parsing the json column - options : dict, optional - options to control parsing. accepts the same options as the json datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a new column of complex type from given JSON object. - Returns a column that evaluates to a struct, array, or map. + variance of given column. + + See Also + -------- + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.var_samp` + :meth:`pyspark.sql.functions.stddev_pop` Examples -------- - Example 1: Parsing JSON with a specified schema + >>> from pyspark.sql import functions as sf + >>> df = spark.range(6) + >>> df.select(sf.var_pop(df.id)).show() + +------------------+ + | var_pop(id)| + +------------------+ + |2.9166666666666...| + +------------------+ + """ + return _invoke_function_over_columns("var_pop", col) - >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import StructType, StructField, IntegerType - >>> schema = StructType([StructField("a", IntegerType())]) - >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, schema).alias("json")).show() - +----+ - |json| - +----+ - | {1}| - +----+ - Example 2: Parsing JSON with a DDL-formatted string. +@_try_remote_functions +def regr_avgx(y: "ColumnOrName", x: "ColumnOrName") -> Column: + """ + Aggregate function: returns the average of the independent variable for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. + + Returns + ------- + :class:`~pyspark.sql.Column` + the average of the independent variable for non-null pairs in a group. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` + + Examples + -------- + Example 1: All pairs are non-null >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, "a INT").alias("json")).show() - +----+ - |json| - +----+ - | {1}| - +----+ + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | 2.75| 2.75| + +---------------+------+ - Example 3: Parsing JSON into a MapType + Example 2: All pairs' x values are null >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, "MAP").alias("json")).show() - +--------+ - | json| - +--------+ - |{a -> 1}| - +--------+ + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | NULL| NULL| + +---------------+------+ - Example 4: Parsing JSON into an ArrayType of StructType + Example 3: All pairs' y values are null >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import ArrayType, StructType, StructField, IntegerType - >>> schema = ArrayType(StructType([StructField("a", IntegerType())])) - >>> df = spark.createDataFrame([(1, '''[{"a": 1}]''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, schema).alias("json")).show() - +-----+ - | json| - +-----+ - |[{1}]| - +-----+ + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | NULL| 1.0| + +---------------+------+ - Example 5: Parsing JSON into an ArrayType + Example 4: Some pairs' x values are null >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType - >>> schema = ArrayType(IntegerType()) - >>> df = spark.createDataFrame([(1, '''[1, 2, 3]''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, schema).alias("json")).show() - +---------+ - | json| - +---------+ - |[1, 2, 3]| - +---------+ + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | 3.0| 3.0| + +---------------+------+ - Example 6: Parsing JSON with specified options + Example 5: Some pairs' x or y values are null >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '''{a:123}'''), (2, '''{"a":456}''')], ("key", "value")) - >>> parsed1 = sf.from_json(df.value, "a INT") - >>> parsed2 = sf.from_json(df.value, "a INT", {"allowUnquotedFieldNames": "true"}) - >>> df.select("value", parsed1, parsed2).show() - +---------+----------------+----------------+ - | value|from_json(value)|from_json(value)| - +---------+----------------+----------------+ - | {a:123}| {NULL}| {123}| - |{"a":456}| {456}| {456}| - +---------+----------------+----------------+ + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | 3.0| 3.0| + +---------------+------+ """ - from pyspark.sql.classic.column import _to_java_column - - if isinstance(schema, DataType): - schema = schema.json() - elif isinstance(schema, Column): - schema = _to_java_column(schema) - return _invoke_function("from_json", _to_java_column(col), schema, _options_to_str(options)) + return _invoke_function_over_columns("regr_avgx", y, x) @_try_remote_functions -def try_parse_json( - col: "ColumnOrName", -) -> Column: +def regr_avgy(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Parses a column containing a JSON string into a :class:`VariantType`. Returns None if a string - contains an invalid JSON value. + Aggregate function: returns the average of the dependent variable for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings. - A column that evaluates to a string. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a new column of VariantType. - Returns a column that evaluates to a variant. + the average of the dependent variable for non-null pairs in a group. - Examples + See Also -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''}, {'json': '''{a : 1}'''} ]) - >>> df.select(to_json(try_parse_json(df.json))).collect() - [Row(to_json(try_parse_json(json))='{"a":1}'), Row(to_json(try_parse_json(json))=None)] - """ - from pyspark.sql.classic.column import _to_java_column + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` - return _invoke_function("try_parse_json", _to_java_column(col)) + Examples + -------- + Example 1: All pairs are non-null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------+ + |regr_avgy(y, x)|avg(y)| + +---------------+------+ + | 1.75| 1.75| + +---------------+------+ + + Example 2: All pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------+ + |regr_avgy(y, x)|avg(y)| + +---------------+------+ + | NULL| 1.0| + +---------------+------+ + + Example 3: All pairs' y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------+ + |regr_avgy(y, x)|avg(y)| + +---------------+------+ + | NULL| NULL| + +---------------+------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +------------------+------+ + | regr_avgy(y, x)|avg(y)| + +------------------+------+ + |1.6666666666666...| 1.75| + +------------------+------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------------------+ + |regr_avgy(y, x)| avg(y)| + +---------------+------------------+ + | 1.5|1.6666666666666...| + +---------------+------------------+ + """ + return _invoke_function_over_columns("regr_avgy", y, x) @_try_remote_functions -def to_variant_object( - col: "ColumnOrName", -) -> Column: +def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Converts a column containing nested inputs (array/map/struct) into a variants where maps and - structs are converted to variant objects which are unordered unlike SQL structs. Input maps can - only have string keys. + Aggregate function: returns the number of non-null number pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column with a nested schema or column name - A column that evaluates to an array, map, or struct. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Returns ------- :class:`~pyspark.sql.Column` - a new column of VariantType. - Returns a column that evaluates to a variant. + the number of non-null number pairs in a group. Examples -------- - Example 1: Converting an array containing a nested struct into a variant + Example 1: All pairs are non-null - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StructType, StructField, StringType, MapType - >>> schema = StructType([ - ... StructField("i", StringType(), True), - ... StructField("v", ArrayType(StructType([ - ... StructField("a", MapType(StringType(), StringType()), True) - ... ]), True)) - ... ]) - >>> data = [("1", [{"a": {"b": 2}}])] - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.to_variant_object(df.v)) - DataFrame[to_variant_object(v): variant] - >>> df.select(sf.to_variant_object(df.v)).show(truncate=False) - +--------------------+ - |to_variant_object(v)| - +--------------------+ - |[{"a":{"b":"2"}}] | - +--------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 4| 4| + +----------------+--------+ - return _invoke_function("to_variant_object", _to_java_column(col)) + Example 2: All pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 0| 1| + +----------------+--------+ + + Example 3: All pairs' y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 0| 1| + +----------------+--------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 3| 4| + +----------------+--------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 2| 4| + +----------------+--------+ + """ + return _invoke_function_over_columns("regr_count", y, x) @_try_remote_functions -def variant_from_arrays(keys: "ColumnOrName", values: "ColumnOrName") -> Column: +def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Creates a variant object from the given arrays of keys and values. The keys must be non-null - strings and the two arrays must have the same length. + Aggregate function: returns the intercept of the univariate linear regression line + for non-null pairs in a group, where `y` is the dependent variable and + `x` is the independent variable. - .. versionadded:: 4.4.0 + .. versionadded:: 3.5.0 Parameters ---------- - keys : :class:`~pyspark.sql.Column` or column name - an array of string keys. - values : :class:`~pyspark.sql.Column` or column name - an array of values. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a new column of VariantType. + the intercept of the univariate linear regression line for non-null pairs in a group. See Also -------- - :meth:`pyspark.sql.functions.variant_from_entries` - :meth:`pyspark.sql.functions.to_variant_object` + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array('a', 'b') AS keys, array(1, 2) AS values") - >>> df.select(sf.variant_from_arrays("keys", "values").cast("string").alias("r")).collect() - [Row(r='{"a":1,"b":2}')] + Example 1: All pairs are non-null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | 0.0| + +--------------------+ + + Example 2: All pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | NULL| + +--------------------+ + + Example 3: All pairs' y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | NULL| + +--------------------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | 0.0| + +--------------------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | 0.0| + +--------------------+ """ - return _invoke_function_over_columns("variant_from_arrays", keys, values) + return _invoke_function_over_columns("regr_intercept", y, x) @_try_remote_functions -def variant_from_entries(entries: "ColumnOrName") -> Column: +def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Creates a variant object from an array of key/value struct entries. The keys must be non-null - strings. + Aggregate function: returns the coefficient of determination for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.4.0 + .. versionadded:: 3.5.0 Parameters ---------- - entries : :class:`~pyspark.sql.Column` or column name - an array of key/value structs, where the first field is a string key and the second field - is the value. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a new column of VariantType. + the coefficient of determination for non-null pairs in a group. See Also -------- - :meth:`pyspark.sql.functions.variant_from_arrays` - :meth:`pyspark.sql.functions.to_variant_object` + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array(struct('a', 1), struct('b', 2)) AS entries") - >>> df.select(sf.variant_from_entries("entries").cast("string").alias("r")).collect() - [Row(r='{"a":1,"b":2}')] - """ - return _invoke_function_over_columns("variant_from_entries", entries) + Example 1: All pairs are non-null + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | 1.0| + +-------------+ -@_try_remote_functions -def parse_json( - col: "ColumnOrName", -) -> Column: - """ - Parses a column containing a JSON string into a :class:`VariantType`. Throws exception if a - string represents an invalid JSON value. + Example 2: All pairs' x values are null - .. versionadded:: 4.0.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | NULL| + +-------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings. - A column that evaluates to a string. + Example 3: All pairs' y values are null - Returns - ------- - :class:`~pyspark.sql.Column` - a new column of VariantType. - Returns a column that evaluates to a variant. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | NULL| + +-------------+ - Examples - -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(to_json(parse_json(df.json))).collect() - [Row(to_json(parse_json(json))='{"a":1}')] - """ - from pyspark.sql.classic.column import _to_java_column + Example 4: Some pairs' x values are null - return _invoke_function("parse_json", _to_java_column(col)) + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | 1.0| + +-------------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | 1.0| + +-------------+ + """ + return _invoke_function_over_columns("regr_r2", y, x) @_try_remote_functions -def is_variant_null(v: "ColumnOrName") -> Column: +def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Check if a variant value is a variant null. Returns true if and only if the input is a variant - null and false otherwise (including in the case of SQL NULL). + Aggregate function: returns the slope of the linear regression line for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a boolean column indicating whether the variant value is a variant null - Returns a column that evaluates to a boolean. + the slope of the linear regression line for non-null pairs in a group. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(is_variant_null(parse_json(df.json)).alias("r")).collect() - [Row(r=False)] - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: All pairs are non-null - return _invoke_function("is_variant_null", _to_java_column(v)) + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | 1.0| + +----------------+ + Example 2: All pairs' x values are null -@_try_remote_functions -def is_valid_variant(v: "ColumnOrName") -> Column: - """ - Check if a variant value is valid. Returns true if the variant is valid, false if it is - malformed, and NULL if the input is NULL. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | NULL| + +----------------+ - .. versionadded:: 4.2.0 + Example 3: All pairs' y values are null - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | NULL| + +----------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - a boolean column indicating whether the variant value is valid - Returns a column that evaluates to a boolean. + Example 4: Some pairs' x values are null - Examples - -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(is_valid_variant(parse_json(df.json)).alias("r")).collect() - [Row(r=True)] - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | 1.0| + +----------------+ - return _invoke_function("is_valid_variant", _to_java_column(v)) + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | 1.0| + +----------------+ + """ + return _invoke_function_over_columns("regr_slope", y, x) @_try_remote_functions -def variant_delete(v: "ColumnOrName", *paths: Union[Column, str]) -> Column: +def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Removes fields or array elements from a variant at the given JSONPath locations. - Multiple paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are - skipped. + Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 5.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - paths : :class:`~pyspark.sql.Column` or str - one or more JSONPath deletion targets. A `str` is a literal path; a - :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path - should start with `$` and is followed by one or more segments like - `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not - A column that evaluates to a string. - allowed. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a variant column with the specified paths removed - Returns a column that evaluates to a variant. + REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_delete - >>> df = spark.createDataFrame([{ - ... 'json': '''{ "a" : 1, "b" : 2, "c" : 3, "items" : [1, 2, 3] }''', - ... 'path': '$.a' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_delete(v, lit(None), "$.a", "$.c")).alias("r")).collect() - [Row(r='{"b":2,"items":[1,2,3]}')] - >>> df.select(to_json(variant_delete(v, "$.missing")).alias("r")).collect() - [Row(r='{"a":1,"b":2,"c":3,"items":[1,2,3]}')] - >>> df.select(to_json(variant_delete(v, df.path)).alias("r")).collect() - [Row(r='{"b":2,"c":3,"items":[1,2,3]}')] - >>> df.select(to_json(variant_delete(v, "$.items[0]", "$.items[0]")).alias("r")).collect() - [Row(r='{"a":1,"b":2,"c":3,"items":[3]}')] - >>> df.select(variant_delete(lit(None), "$.a").alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + Example 1: All pairs are non-null - if len(paths) == 0: - raise PySparkValueError( - errorClass="CANNOT_BE_EMPTY", - messageParameters={"item": "paths"}, - ) - sc = _get_active_spark_context() + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | 5.0| + +--------------+ - path_cols = [p if isinstance(p, Column) else lit(p) for p in paths] - return _invoke_function( - "variant_delete", - _to_java_column(v), - _to_java_column(path_cols[0]), - _to_seq(sc, path_cols[1:], _to_java_column), - ) + Example 2: All pairs' x values are null + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | NULL| + +--------------+ -@_try_remote_functions -def variant_insert(v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName") -> Column: - """ - Inserts a value into a variant at the given JSONPath location. An object path adds a new field - (error if it already exists); an array path inserts at the index, shifting later elements - right. Missing intermediate keys are created. Throws an error if a path segment hits a value - of an incompatible type. Returns NULL if any argument is NULL. + Example 3: All pairs' y values are null - .. versionadded:: 4.3.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | NULL| + +--------------+ - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath insertion target. A `str` is a literal path; a - :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with - `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or - A column that evaluates to a string. - `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to insert. Any expression castable to variant. + Example 4: Some pairs' x values are null - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with `value` inserted at `path` - Returns a column that evaluates to a variant. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +-----------------+ + | regr_sxx(y, x)| + +-----------------+ + |4.666666666666...| + +-----------------+ - Examples - -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_insert - >>> df = spark.createDataFrame([{ - ... 'json': '''{ "a": 1, "arr": ["x", "y"] }''', - ... 'path': '$.d' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_insert(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"b":2}')] - >>> df.select(to_json(variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] - >>> df.select(to_json(variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","z","y"]}')] - >>> df.select(to_json(variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] - >>> df.select(to_json(variant_insert(v, df.path, lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"d":9}')] - >>> df.select(to_json(variant_insert(v, "$.b", parse_json(lit('null')))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"b":null}')] - >>> df.select(variant_insert(v, "$.b", lit(None)).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column + Example 5: Some pairs' x or y values are null - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "variant_insert", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | 4.5| + +--------------+ + """ + return _invoke_function_over_columns("regr_sxx", y, x) @_try_remote_functions -def try_variant_insert( - v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" -) -> Column: +def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Inserts a value into a variant at the given JSONPath location. An object path adds a new field; - an array path inserts at the index, shifting later elements right. Missing intermediate keys - are created. Returns NULL if the field already exists or a path segment hits a value of an - incompatible type, or if any argument is NULL. + Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath insertion target. A `str` is a literal path; a - :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with - `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or - `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to insert. Any expression castable to variant. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_syy` Returns ------- :class:`~pyspark.sql.Column` - a variant column with `value` inserted at `path`, or NULL if the insertion fails - Returns a column that evaluates to a variant. + REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group. Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_insert - >>> df = spark.createDataFrame([{'json': '''{ "a": 1, "arr": ["x", "y"] }'''}]) - >>> v = parse_json(df.json) - >>> df.select(to_json(try_variant_insert(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"b":2}')] - >>> df.select(to_json(try_variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] - >>> df.select(to_json(try_variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","z","y"]}')] - >>> df.select(to_json(try_variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] - >>> df.select(to_json(try_variant_insert(v, "$.a", lit(2))).alias("r")).collect() - [Row(r=None)] - >>> df.select(to_json(try_variant_insert(v, "$.a.b", lit(2))).alias("r")).collect() - [Row(r=None)] - >>> df.select(to_json(try_variant_insert(v, "$.b", lit(None))).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: All pairs are non-null - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "try_variant_insert", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | 5.0| + +--------------+ + + Example 2: All pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 3: All pairs' y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +-----------------+ + | regr_sxy(y, x)| + +-----------------+ + |4.666666666666...| + +-----------------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | 4.5| + +--------------+ + """ + return _invoke_function_over_columns("regr_sxy", y, x) @_try_remote_functions -def variant_set( - v: "ColumnOrName", - path: Union[Column, str], - value: "ColumnOrName", - create_if_missing: bool = True, -) -> Column: +def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Sets or upserts a value in a variant at the given JSONPath location. An existing object field - or array element at the target is replaced. A missing field, array index, or intermediate path - is created, unless `create_if_missing` is false, in which case the variant is left unchanged. - Throws an error if a path segment hits a value of an incompatible type. Returns NULL if any - argument is NULL. + Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies - the path at runtime. A valid path should start with `$` and is followed by one or more - A column that evaluates to a string. - segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to set. Any expression castable to variant. - create_if_missing : bool, optional - whether to create missing keys or out-of-range array indices (default True). - A column that evaluates to a boolean. Must be a constant. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a variant column with `value` set at `path` - Returns a column that evaluates to a variant. + REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_set - >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_set(v, "$.a", lit(9))).alias("r")).collect() - [Row(r='{"a":9,"arr":[1,2,3]}')] - >>> df.select(to_json(variant_set(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] - >>> df.select(to_json(variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,9,3]}')] - >>> df.select(to_json(variant_set(v, "$.b", lit(2), False)).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3]}')] - >>> df.select(to_json(variant_set(v, "$.a", parse_json(lit("null")))).alias("r")).collect() - [Row(r='{"a":null,"arr":[1,2,3]}')] - >>> df.select(to_json(variant_set(v, "$.a", lit(None))).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: All pairs are non-null - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "variant_set", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - _enum_to_value(create_if_missing), - ) + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | 5.0| + +--------------+ + + Example 2: All pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 3: All pairs' y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +-----------------+ + | regr_syy(y, x)| + +-----------------+ + |4.666666666666...| + +-----------------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | 4.5| + +--------------+ + """ + return _invoke_function_over_columns("regr_syy", y, x) @_try_remote_functions -def try_variant_set( - v: "ColumnOrName", - path: Union[Column, str], - value: "ColumnOrName", - create_if_missing: bool = True, -) -> Column: +def every(col: "ColumnOrName") -> Column: """ - Sets or upserts a value in a variant at the given JSONPath location. An existing object field - or array element at the target is replaced. A missing field, array index, or intermediate path - is created, unless `create_if_missing` is false, in which case the variant is left unchanged. - Returns NULL if a path segment hits a value of an incompatible type, or if any argument is NULL. + Aggregate function: returns true if all values of `col` are true. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - path : :class:`~pyspark.sql.Column` or str - the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies - the path at runtime. A valid path should start with `$` and is followed by one or more - segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to set. Any expression castable to variant. - create_if_missing : bool, optional - whether to create missing keys or out-of-range array indices (default True). + col : :class:`~pyspark.sql.Column` or column name + column to check if all values are true. + A column that evaluates to a boolean. + + See Also + -------- + :meth:`pyspark.sql.functions.some` Returns ------- :class:`~pyspark.sql.Column` - a variant column with `value` set at `path` + true if all values of `col` are true, false otherwise. Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_set - >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) - >>> v = parse_json(df.json) - >>> df.select(to_json(try_variant_set(v, "$.a", lit(9))).alias("r")).collect() - [Row(r='{"a":9,"arr":[1,2,3]}')] - >>> df.select(to_json(try_variant_set(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] - >>> df.select(to_json(try_variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,9,3]}')] - >>> df.select(to_json(try_variant_set(v, "$.arr[5]", lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3,null,null,9]}')] - >>> df.select(to_json(try_variant_set(v, "$.b", lit(2), False)).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3]}')] - >>> df.select(to_json(try_variant_set(v, "$.a.b", lit(9))).alias("r")).collect() - [Row(r=None)] - >>> df.select(to_json(try_variant_set(v, "$.a", lit(None))).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [True], [True]], ["flag"] + ... ).select(sf.every("flag")).show() + +-----------+ + |every(flag)| + +-----------+ + | true| + +-----------+ - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "try_variant_set", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - _enum_to_value(create_if_missing), - ) + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [False], [True]], ["flag"] + ... ).select(sf.every("flag")).show() + +-----------+ + |every(flag)| + +-----------+ + | false| + +-----------+ + + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[False], [False], [False]], ["flag"] + ... ).select(sf.every("flag")).show() + +-----------+ + |every(flag)| + +-----------+ + | false| + +-----------+ + """ + return _invoke_function_over_columns("every", col) @_try_remote_functions -def variant_array_append( - v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" -) -> Column: +def bool_and(col: "ColumnOrName") -> Column: """ - Appends a value to the array in a variant at the given JSONPath location. Returns the variant - unchanged if a path key or index is absent. Throws an error if a path segment hits a value of - an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + Aggregate function: returns true if all values of `col` are true. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` - supplies the path at runtime. A valid path should start with `$` and is followed by zero or - A column that evaluates to a string. - more segments like `[123]`, `.name`, `['name']`, or `["name"]`. - value : :class:`~pyspark.sql.Column` or str - the value to append. Any expression castable to variant. + col : :class:`~pyspark.sql.Column` or column name + column to check if all values are true. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - a variant column with `value` appended to the array at `path` - Returns a column that evaluates to a variant. + true if all values of `col` are true, false otherwise. + + See Also + -------- + :meth:`pyspark.sql.functions.bool_or` Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_array_append - >>> df = spark.createDataFrame([{ - ... 'json': '''[[1, 2], 5]''', - ... 'path': '$[0]' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_array_append(v, "$", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5,3]')] - >>> df.select(to_json(variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5]')] - >>> df.select(to_json(variant_array_append(v, df.path, lit(9))).alias("r")).collect() - [Row(r='[[1,2,9],5]')] - >>> nested = variant_array_append(v, "$", parse_json(lit('[4, 5]'))) - >>> df.select(to_json(nested).alias("r")).collect() - [Row(r='[[1,2],5,[4,5]]')] - >>> df.select(variant_array_append(v, "$", lit(None)).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) + >>> df.select(sf.bool_and("flag")).show() + +--------------+ + |bool_and(flag)| + +--------------+ + | true| + +--------------+ - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "variant_array_append", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) + >>> df.select(sf.bool_and("flag")).show() + +--------------+ + |bool_and(flag)| + +--------------+ + | false| + +--------------+ + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) + >>> df.select(sf.bool_and("flag")).show() + +--------------+ + |bool_and(flag)| + +--------------+ + | false| + +--------------+ + """ + return _invoke_function_over_columns("bool_and", col) @_try_remote_functions -def try_variant_array_append( - v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" -) -> Column: +def some(col: "ColumnOrName") -> Column: """ - Appends a value to the array in a variant at the given JSONPath location. Returns the variant - unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an - incompatible type, the target is not an array, or if any argument is NULL. + Aggregate function: returns true if at least one value of `col` is true. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - path : :class:`~pyspark.sql.Column` or str - the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` - supplies the path at runtime. A valid path should start with `$` and is followed by zero or - more segments like `[123]`, `.name`, `['name']`, or `["name"]`. - value : :class:`~pyspark.sql.Column` or str - the value to append. Any expression castable to variant. + col : :class:`~pyspark.sql.Column` or column name + column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - a variant column with `value` appended to the array at `path` + true if at least one value of `col` is true, false otherwise. + + See Also + -------- + :meth:`pyspark.sql.functions.every` Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_array_append - >>> df = spark.createDataFrame([{ - ... 'json': '''[[1, 2], 5]''', - ... 'path': '$[0]' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(try_variant_array_append(v, "$", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5,3]')] - >>> df.select(to_json(try_variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5]')] - >>> df.select(to_json(try_variant_array_append(v, df.path, lit(9))).alias("r")).collect() - [Row(r='[[1,2,9],5]')] - >>> df.select(to_json(try_variant_array_append(v, "$[1]", lit(9))).alias("r")).collect() - [Row(r=None)] - >>> df.select(try_variant_array_append(v, "$", lit(None)).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [True], [True]], ["flag"] + ... ).select(sf.some("flag")).show() + +----------+ + |some(flag)| + +----------+ + | true| + +----------+ - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "try_variant_array_append", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [False], [True]], ["flag"] + ... ).select(sf.some("flag")).show() + +----------+ + |some(flag)| + +----------+ + | true| + +----------+ + + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[False], [False], [False]], ["flag"] + ... ).select(sf.some("flag")).show() + +----------+ + |some(flag)| + +----------+ + | false| + +----------+ + """ + return _invoke_function_over_columns("some", col) @_try_remote_functions -def variant_strip_nulls(v: "ColumnOrName", include_arrays: bool = True) -> Column: +def bool_or(col: "ColumnOrName") -> Column: """ - Recursively removes object fields and array elements whose value is a variant null, unless - `include_arrays` is False, in which case null array elements are kept. Returns NULL if any - argument is NULL. + Aggregate function: returns true if at least one value of `col` is true. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - include_arrays : bool, optional - whether null elements are also removed from arrays (default True). + col : :class:`~pyspark.sql.Column` or column name + column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - a variant column with variant null fields/elements removed + true if at least one value of `col` is true, false otherwise. + + See Also + -------- + :meth:`pyspark.sql.functions.bool_and` Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_strip_nulls - >>> df = spark.createDataFrame([{ - ... 'json': '''{ "a" : 1, "b" : null, "c" : [1, null], "d" : { "e" : null, "f" : 4 } }''' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_strip_nulls(v)).alias("r")).collect() - [Row(r='{"a":1,"c":[1],"d":{"f":4}}')] - >>> df.select(to_json(variant_strip_nulls(v, False)).alias("r")).collect() - [Row(r='{"a":1,"c":[1,null],"d":{"f":4}}')] - >>> df.select(variant_strip_nulls(lit(None)).alias("r")).collect() - [Row(r=None)] - >>> df2 = spark.createDataFrame([{'json': '{"a": null}'}, {'json': 'null'}]) - >>> v2 = parse_json(df2.json) - >>> df2.select(to_json(variant_strip_nulls(v2)).alias("r")).collect() - [Row(r='{}'), Row(r='null')] + >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) + >>> df.select(bool_or("flag")).show() + +-------------+ + |bool_or(flag)| + +-------------+ + | true| + +-------------+ + >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) + >>> df.select(bool_or("flag")).show() + +-------------+ + |bool_or(flag)| + +-------------+ + | true| + +-------------+ + >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) + >>> df.select(bool_or("flag")).show() + +-------------+ + |bool_or(flag)| + +-------------+ + | false| + +-------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "variant_strip_nulls", _to_java_column(v), _enum_to_value(include_arrays) - ) + return _invoke_function_over_columns("bool_or", col) @_try_remote_functions -def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: +def bit_and(col: "ColumnOrName") -> Column: """ - Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to - `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. + Aggregate function: returns the bitwise AND of all non-null input values, or null if none. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - a column containing the extraction path strings or a string representing the extraction - path. A valid path should start with `$` and is followed by zero or more segments like - A column that evaluates to a string. - `[123]`, `.name`, `['name']`, or `["name"]`. - targetType : str - A DDL-formatted type string. Must be a constant. - the target data type to cast into, in a DDL-formatted string + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` - a column of `targetType` representing the extracted result - Returns a column of the type given by `targetType`. + the bitwise AND of all non-null input values, or null if none. + + See Also + -------- + :meth:`pyspark.sql.functions.bit_or` + :meth:`pyspark.sql.functions.bit_xor` Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) - >>> df.select(variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() - [Row(r=1)] - >>> df.select(variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() - [Row(r=None)] - >>> df.select(variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() - [Row(r=1)] - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: Bitwise AND with all non-null values - assert isinstance(path, (Column, str)) - if isinstance(path, str): - return _invoke_function( - "variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) - ) - else: - return _invoke_function( - "variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) - ) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | 0| + +----------+ + + Example 2: Bitwise AND with null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | 0| + +----------+ + + Example 3: Bitwise AND with all null values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([StructField("c", IntegerType(), True)]) + >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | NULL| + +----------+ + + Example 4: Bitwise AND with single input value + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[5]], ["c"]) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | 5| + +----------+ + """ + return _invoke_function_over_columns("bit_and", col) @_try_remote_functions -def try_variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: +def bit_or(col: "ColumnOrName") -> Column: """ - Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to - `targetType`. Returns null if the path does not exist or the cast fails. + Aggregate function: returns the bitwise OR of all non-null input values, or null if none. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - a column containing the extraction path strings or a string representing the extraction - path. A valid path should start with `$` and is followed by zero or more segments like - A column that evaluates to a string. - `[123]`, `.name`, `['name']`, or `["name"]`. - targetType : str - A DDL-formatted type string. Must be a constant. - the target data type to cast into, in a DDL-formatted string + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` - a column of `targetType` representing the extracted result - Returns a column of the type given by `targetType`. + the bitwise OR of all non-null input values, or null if none. + + See Also + -------- + :meth:`pyspark.sql.functions.bit_and` + :meth:`pyspark.sql.functions.bit_xor` Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) - >>> df.select(try_variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() - [Row(r=1)] - >>> df.select(try_variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() - [Row(r=None)] - >>> df.select(try_variant_get(parse_json(df.json), "$.a", "binary").alias("r")).collect() - [Row(r=None)] - >>> df.select(try_variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() - [Row(r=1)] - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: Bitwise OR with all non-null values - if isinstance(path, str): - return _invoke_function( - "try_variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) - ) - else: - return _invoke_function( - "try_variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) - ) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | 3| + +---------+ + + Example 2: Bitwise OR with some null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | 3| + +---------+ + + Example 3: Bitwise OR with all null values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([StructField("c", IntegerType(), True)]) + >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | NULL| + +---------+ + + Example 4: Bitwise OR with single input value + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[5]], ["c"]) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | 5| + +---------+ + """ + return _invoke_function_over_columns("bit_or", col) @_try_remote_functions -def schema_of_variant(v: "ColumnOrName") -> Column: +def bit_xor(col: "ColumnOrName") -> Column: """ - Returns schema in the SQL format of a variant. + Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` - a string column representing the variant schema - Returns a column that evaluates to a string. + the bitwise XOR of all non-null input values, or null if none. + + See Also + -------- + :meth:`pyspark.sql.functions.bit_and` + :meth:`pyspark.sql.functions.bit_or` Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(schema_of_variant(parse_json(df.json)).alias("r")).collect() - [Row(r='OBJECT')] - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: Bitwise XOR with all non-null values - return _invoke_function("schema_of_variant", _to_java_column(v)) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | 2| + +----------+ + + Example 2: Bitwise XOR with some null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | 3| + +----------+ + + Example 3: Bitwise XOR with all null values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([StructField("c", IntegerType(), True)]) + >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | NULL| + +----------+ + + Example 4: Bitwise XOR with single input value + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[5]], ["c"]) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | 5| + +----------+ + """ + return _invoke_function_over_columns("bit_xor", col) @_try_remote_functions -def schema_of_variant_agg(v: "ColumnOrName") -> Column: +def skewness(col: "ColumnOrName") -> Column: """ - Returns the merged schema in the SQL format of a variant column. + Aggregate function: returns the skewness of the values in a group. - .. versionadded:: 4.0.0 + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. + + See Also + -------- + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.kurtosis` Returns ------- :class:`~pyspark.sql.Column` - a string column representing the variant schema - Returns a column that evaluates to a string. + skewness of given column. Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(schema_of_variant_agg(parse_json(df.json)).alias("r")).collect() - [Row(r='OBJECT')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.skewness(df.c)).show() + +------------------+ + | skewness(c)| + +------------------+ + |0.7071067811865...| + +------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("schema_of_variant_agg", _to_java_column(v)) + return _invoke_function_over_columns("skewness", col) @_try_remote_functions -def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: +def kurtosis(col: "ColumnOrName") -> Column: """ - Converts a column containing a :class:`StructType`, :class:`ArrayType`, :class:`MapType` - or a :class:`VariantType` into a JSON string. Throws an exception, in the case of an unsupported type. + Aggregate function: returns the kurtosis of the values in a group. - .. versionadded:: 2.1.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing a struct, an array, a map, or a variant object. - A column that evaluates to a struct, array, map, or variant. - options : dict, optional - options to control converting. accepts the same options as the JSON datasource. - See `Data Source Option `_ - for the version you use. - Additionally the function supports the `pretty` option which enables - A dict of options. Each key and value is a string. - pretty JSON generation. - - .. # noqa + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - JSON object as string column. - Returns a column that evaluates to a string. + kurtosis of given column. - Examples + See Also -------- - Example 1: Converting a StructType column to JSON - - >>> import pyspark.sql.functions as sf - >>> from pyspark.sql import Row - >>> data = [(1, Row(age=2, name='Alice'))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +------------------------+ - |json | - +------------------------+ - |{"age":2,"name":"Alice"}| - +------------------------+ - - Example 2: Converting an ArrayType column to JSON - - >>> import pyspark.sql.functions as sf - >>> from pyspark.sql import Row - >>> data = [(1, [Row(age=2, name='Alice'), Row(age=3, name='Bob')])] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +-------------------------------------------------+ - |json | - +-------------------------------------------------+ - |[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]| - +-------------------------------------------------+ - - Example 3: Converting a MapType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, {"name": "Alice"})], ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +----------------+ - |json | - +----------------+ - |{"name":"Alice"}| - +----------------+ - - Example 4: Converting a VariantType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '{"name": "Alice"}')], ("key", "value")) - >>> df.select(sf.to_json(sf.parse_json(df.value)).alias("json")).show(truncate=False) - +----------------+ - |json | - +----------------+ - |{"name":"Alice"}| - +----------------+ - - Example 5: Converting a nested MapType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, [{"name": "Alice"}, {"name": "Bob"}])], ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +---------------------------------+ - |json | - +---------------------------------+ - |[{"name":"Alice"},{"name":"Bob"}]| - +---------------------------------+ - - Example 6: Converting a simple ArrayType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, ["Alice", "Bob"])], ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +---------------+ - |json | - +---------------+ - |["Alice","Bob"]| - +---------------+ - - Example 7: Converting to JSON with specified options + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.skewness` - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT (DATE('2022-02-22'), 1) AS date") - >>> json1 = sf.to_json(df.date) - >>> json2 = sf.to_json(df.date, {"dateFormat": "yyyy/MM/dd"}) - >>> df.select("date", json1, json2).show(truncate=False) - +---------------+------------------------------+------------------------------+ - |date |to_json(date) |to_json(date) | - +---------------+------------------------------+------------------------------+ - |{2022-02-22, 1}|{"col1":"2022-02-22","col2":1}|{"col1":"2022/02/22","col2":1}| - +---------------+------------------------------+------------------------------+ + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.kurtosis(df.c)).show() + +-----------+ + |kurtosis(c)| + +-----------+ + | -1.5| + +-----------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("to_json", _to_java_column(col), _options_to_str(options)) + return _invoke_function_over_columns("kurtosis", col) @_try_remote_functions -def schema_of_json(json: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: +def collect_list(col: "ColumnOrName") -> Column: """ - Parses a JSON string and infers its schema in DDL format. + Aggregate function: Collects the values from a column into a list, + maintaining duplicates, and returns this list of objects. - .. versionadded:: 2.4.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - json : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a JSON string or a foldable string column containing a JSON string. - options : dict, optional - options to control parsing. accepts the same options as the JSON datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col : :class:`~pyspark.sql.Column` or column name + The target column on which the function is computed. - .. versionchanged:: 3.0.0 - It accepts `options` parameter to control schema inferring. + See Also + -------- + :meth:`pyspark.sql.functions.array_agg` + :meth:`pyspark.sql.functions.collect_set` Returns ------- :class:`~pyspark.sql.Column` - a string representation of a :class:`StructType` parsed from given JSON. - Returns a column that evaluates to a string. + A new Column object representing a list of collected values, with duplicate values included. + + Notes + ----- + The function is non-deterministic as the order of collected results depends + on the order of the rows, which possibly becomes non-deterministic after shuffle operations. Examples -------- - >>> import pyspark.sql.functions as sf - >>> parsed1 = sf.schema_of_json(sf.lit('{"a": 0}')) - >>> parsed2 = sf.schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'}) - >>> spark.range(1).select(parsed1, parsed2).show() - +------------------------+----------------------+ - |schema_of_json({"a": 0})|schema_of_json({a: 1})| - +------------------------+----------------------+ - | STRUCT| STRUCT| - +------------------------+----------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: Collect values from a DataFrame and sort the result in ascending order - json = _enum_to_value(json) - if not isinstance(json, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "json", - "arg_type": type(json).__name__, - }, - ) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) + >>> df.select(sf.sort_array(sf.collect_list('value')).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [1, 2, 2]| + +-----------+ - return _invoke_function("schema_of_json", _to_java_column(lit(json)), _options_to_str(options)) + Example 2: Collect values from a DataFrame and sort the result in descending order + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) + >>> df.select(sf.sort_array(sf.collect_list('age'), asc=False).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [5, 5, 2]| + +-----------+ + + Example 3: Collect values from a DataFrame with multiple columns and sort the result + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) + >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_list('id')).alias('sorted_list')) + >>> df.orderBy(sf.desc("name")).show() + +----+-----------+ + |name|sorted_list| + +----+-----------+ + |John| [1, 2]| + | Ana| [3]| + +----+-----------+ + """ + return _invoke_function_over_columns("collect_list", col) @_try_remote_functions -def json_array_length(col: "ColumnOrName") -> Column: +def array_agg(col: "ColumnOrName") -> Column: """ - Returns the number of elements in the outermost JSON array. `NULL` is returned in case of - any other valid JSON string, `NULL` or an invalid JSON. + Aggregate function: returns a list of objects with duplicates. .. versionadded:: 3.5.0 Parameters ---------- - col: :class:`~pyspark.sql.Column` or str + col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - length of json array. - Returns a column that evaluates to an integer. + list of objects with duplicates. + + See Also + -------- + :meth:`pyspark.sql.functions.collect_list` + :meth:`pyspark.sql.functions.collect_set` Examples -------- - >>> df = spark.createDataFrame([(None,), ('[1, 2, 3]',), ('[]',)], ['data']) - >>> df.select(json_array_length(df.data).alias('r')).collect() - [Row(r=None), Row(r=3), Row(r=0)] - """ - return _invoke_function_over_columns("json_array_length", col) + Example 1: Using array_agg function on an int column + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [1, 1, 2]| + +-----------+ -@_try_remote_functions -def json_object_keys(col: "ColumnOrName") -> Column: - """ - Returns all the keys of the outermost JSON object as an array. If a valid JSON object is - given, all the keys of the outermost object will be returned as an array. If it is any - other valid JSON string, an invalid JSON string or an empty string, the function returns null. + Example 2: Using array_agg function on a string column - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([["apple"],["apple"],["banana"]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show(truncate=False) + +----------------------+ + |sorted_list | + +----------------------+ + |[apple, apple, banana]| + +----------------------+ - Parameters - ---------- - col: :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to a string. + Example 3: Using array_agg function on a column with null values - Returns - ------- - :class:`~pyspark.sql.Column` - all the keys of the outermost JSON object. - Returns a column that evaluates to an array. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [1, 2]| + +-----------+ - Examples - -------- - >>> df = spark.createDataFrame([(None,), ('{}',), ('{"key1":1, "key2":2}',)], ['data']) - >>> df.select(json_object_keys(df.data).alias('r')).collect() - [Row(r=None), Row(r=[]), Row(r=['key1', 'key2'])] + Example 4: Using array_agg function on a column with different data types + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],["apple"],[2]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() + +-------------+ + | sorted_list| + +-------------+ + |[1, 2, apple]| + +-------------+ """ - return _invoke_function_over_columns("json_object_keys", col) + return _invoke_function_over_columns("array_agg", col) @_try_remote_functions -def json_typeof(col: "ColumnOrName") -> Column: +def collect_set(col: "ColumnOrName") -> Column: """ - Returns the type of the outermost JSON value as a string: one of 'object', 'array', - 'string', 'number', 'boolean', or 'null'. Returns null if the input is not a valid JSON - string or is an empty string. + Aggregate function: Collects the values from a column into a set, + eliminating duplicates, and returns this set of objects. - .. versionadded:: 4.4.0 + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col: :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The target column on which the function is computed. Returns ------- :class:`~pyspark.sql.Column` - the type of the outermost JSON value. - Returns a column that evaluates to a string. + A new Column object representing a set of collected values, duplicates excluded. See Also -------- - :meth:`pyspark.sql.functions.json_object_keys` - :meth:`pyspark.sql.functions.get_json_object` - :meth:`pyspark.sql.functions.json_array_length` + :meth:`pyspark.sql.functions.array_agg` + :meth:`pyspark.sql.functions.collect_list` + + Notes + ----- + This function is non-deterministic as the order of collected results depends + on the order of the rows, which may be non-deterministic after any shuffle operations. Examples -------- - >>> df = spark.createDataFrame([('{"a": 1}',), ('[1, 2, 3]',), ('123',), ('',)], ['data']) - >>> df.select(json_typeof(df.data).alias('r')).collect() - [Row(r='object'), Row(r='array'), Row(r='number'), Row(r=None)] + Example 1: Collect values from a DataFrame and sort the result in ascending order + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) + >>> df.select(sf.sort_array(sf.collect_set('value')).alias('sorted_set')).show() + +----------+ + |sorted_set| + +----------+ + | [1, 2]| + +----------+ + + Example 2: Collect values from a DataFrame and sort the result in descending order + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) + >>> df.select(sf.sort_array(sf.collect_set('age'), asc=False).alias('sorted_set')).show() + +----------+ + |sorted_set| + +----------+ + | [5, 2]| + +----------+ + + Example 3: Collect values from a DataFrame with multiple columns and sort the result + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) + >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_set('id')).alias('sorted_set')) + >>> df.orderBy(sf.desc("name")).show() + +----+----------+ + |name|sorted_set| + +----+----------+ + |John| [1, 2]| + | Ana| [3]| + +----+----------+ """ - return _invoke_function_over_columns("json_typeof", col) + return _invoke_function_over_columns("collect_set", col) -# TODO: Fix and add an example for StructType with Spark Connect -# e.g., StructType([StructField("a", IntegerType())]) @_try_remote_functions -def from_xml( - col: "ColumnOrName", - schema: Union[StructType, Column, str], - options: Optional[Mapping[str, str]] = None, -) -> Column: +def collect_union(col: "ColumnOrName") -> Column: """ - Parses a column containing a XML string to a row with - the specified schema. Returns `null`, in the case of an unparsable string. + Aggregate function: given an array-typed column, collects the distinct union of the + elements of the arrays across rows and returns it as an array. - .. versionadded:: 4.0.0 + The aggregation buffer holds only the distinct elements, so its size is bounded by the + element universe rather than by the number of input rows. Null elements are dropped by + default (``IGNORE NULLS``), matching :func:`collect_set`. With ``RESPECT NULLS`` a single + null element is kept, in which case this is equivalent to + ``array_distinct(flatten(collect_list(col)))``. The ``RESPECT NULLS`` clause is only + available through SQL, e.g. ``expr("collect_union(col) RESPECT NULLS")``. + + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a column or column name in XML format - schema : :class:`StructType`, :class:`~pyspark.sql.Column` or str - a StructType, Column or Python string literal with a DDL-formatted string - A column that evaluates to a string, or a DDL-formatted type string, or a DataType. - to use when parsing the Xml column - options : dict, optional - options to control parsing. accepts the same options as the Xml datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col : :class:`~pyspark.sql.Column` or column name + The target array column on which the function is computed. Returns ------- :class:`~pyspark.sql.Column` - a new column of complex type from given XML object. - Returns a column that evaluates to a struct. + A new Column object representing the distinct union of the array elements. + + See Also + -------- + :meth:`pyspark.sql.functions.collect_set` + :meth:`pyspark.sql.functions.collect_list` + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.flatten` + + Notes + ----- + This function is non-deterministic as the order of collected results depends + on the order of the rows, which may be non-deterministic after any shuffle operations. Examples -------- - Example 1: Parsing XML with a DDL-formatted string schema + Example 1: Union the elements of array columns across rows - >>> import pyspark.sql.functions as sf - >>> data = [(1, '''

1

''')] - >>> df = spark.createDataFrame(data, ("key", "value")) - ... # Define the schema using a DDL-formatted string - >>> schema = "STRUCT" - ... # Parse the XML column using the DDL-formatted schema - >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() - [Row(xml=Row(a=1))] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [([1, 2],), ([2, 3],), ([1],)], ('value',)) + >>> df.select(sf.sort_array(sf.collect_union('value')).alias('u')).show() + +---------+ + | u| + +---------+ + |[1, 2, 3]| + +---------+ - Example 2: Parsing XML with a :class:`StructType` schema + Example 2: Union per group - >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import StructType, LongType - >>> data = [(1, '''

1

''')] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> schema = StructType().add("a", LongType()) - >>> df.select(sf.from_xml(df.value, schema)).show() - +---------------+ - |from_xml(value)| - +---------------+ - | {1}| - +---------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("a", [1, 2]), ("a", [2, 3]), ("b", [4])], ("k", "value")) + >>> df = df.groupBy("k").agg(sf.sort_array(sf.collect_union('value')).alias('u')) + >>> df.orderBy("k").show() + +---+---------+ + | k| u| + +---+---------+ + | a|[1, 2, 3]| + | b| [4]| + +---+---------+ + """ + return _invoke_function_over_columns("collect_union", col) - Example 3: Parsing XML with :class:`ArrayType` in schema - >>> import pyspark.sql.functions as sf - >>> data = [(1, '

12

')] - >>> df = spark.createDataFrame(data, ("key", "value")) - ... # Define the schema with an Array type - >>> schema = "STRUCT>" - ... # Parse the XML column using the schema with an Array - >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() - [Row(xml=Row(a=[1, 2]))] +@_try_remote_functions +def approxCountDistinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: + """ + This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates + the approximate distinct count of elements in a specified column or a group of columns. - Example 4: Parsing XML using :meth:`pyspark.sql.functions.schema_of_xml` + .. versionadded:: 1.3.0 - >>> import pyspark.sql.functions as sf - >>> # Sample data with an XML column - ... data = [(1, '

12

')] - >>> df = spark.createDataFrame(data, ("key", "value")) - ... # Generate the schema from an example XML value - >>> schema = sf.schema_of_xml(sf.lit(data[0][1])) - ... # Parse the XML column using the generated schema - >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() - [Row(xml=Row(a=[1, 2]))] + .. versionchanged:: 3.4.0 + Supports Spark Connect. - See Also - -------- - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` + .. deprecated:: 2.1.0 + Use :func:`approx_count_distinct` instead. """ - from pyspark.sql.classic.column import _to_java_column - - if isinstance(schema, StructType): - schema = schema.json() - elif isinstance(schema, Column): - schema = _to_java_column(schema) - elif not isinstance(schema, str): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "StructType, Column or str", - "arg_name": "schema", - "arg_type": type(schema).__name__, - }, - ) - return _invoke_function("from_xml", _to_java_column(col), schema, _options_to_str(options)) + warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", FutureWarning) + return approx_count_distinct(col, rsd) @_try_remote_functions -def schema_of_xml(xml: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: +def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: """ - Parses a XML string and infers its schema in DDL format. + This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates + the approximate distinct count of elements in a specified column or a group of columns. - .. versionadded:: 4.0.0 + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - xml : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a XML string or a foldable string column containing a XML string. - options : dict, optional - options to control parsing. accepts the same options as the XML datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col : :class:`~pyspark.sql.Column` or column name + The label of the column to count distinct values in. + rsd : float, optional + The maximum allowed relative standard deviation (default = 0.05). + If rsd < 0.01, it would be more efficient to use :func:`count_distinct`. Returns ------- :class:`~pyspark.sql.Column` - a string representation of a :class:`StructType` parsed from given XML. - Returns a column that evaluates to a string. + A new Column object representing the approximate unique count. - Examples + See Also -------- - Example 1: Parsing a simple XML with a single element - - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_xml(sf.lit('

1

')).alias("xml")).collect() - [Row(xml='STRUCT')] + :meth:`pyspark.sql.functions.count_distinct` - Example 2: Parsing an XML with multiple elements in an array + Examples + -------- + Example 1: Counting distinct values in a single column DataFrame representing integers >>> from pyspark.sql import functions as sf - >>> df.select(sf.schema_of_xml(sf.lit('

12

')).alias("xml")).collect() - [Row(xml='STRUCT>')] + >>> df = spark.createDataFrame([1,2,2,3], "int") + >>> df.agg(sf.approx_count_distinct("value")).show() + +----------------------------+ + |approx_count_distinct(value)| + +----------------------------+ + | 3| + +----------------------------+ - Example 3: Parsing XML with options to exclude attributes + Example 2: Counting distinct values in a single column DataFrame representing strings >>> from pyspark.sql import functions as sf - >>> schema = sf.schema_of_xml('

1

', {'excludeAttribute':'true'}) - >>> df.select(schema.alias("xml")).collect() - [Row(xml='STRUCT')] + >>> df = spark.createDataFrame([("apple",), ("orange",), ("apple",), ("banana",)], ['fruit']) + >>> df.agg(sf.approx_count_distinct("fruit")).show() + +----------------------------+ + |approx_count_distinct(fruit)| + +----------------------------+ + | 3| + +----------------------------+ - Example 4: Parsing XML with complex structure + Example 3: Counting distinct values in a DataFrame with multiple columns >>> from pyspark.sql import functions as sf - >>> df.select( - ... sf.schema_of_xml( - ... sf.lit('Alice30') - ... ).alias("xml") - ... ).collect() - [Row(xml='STRUCT>')] + >>> df = spark.createDataFrame( + ... [("Alice", 1), ("Alice", 2), ("Bob", 3), ("Bob", 3)], ["name", "value"]) + >>> df = df.withColumn("combined", sf.struct("name", "value")) + >>> df.agg(sf.approx_count_distinct(df.combined)).show() + +-------------------------------+ + |approx_count_distinct(combined)| + +-------------------------------+ + | 3| + +-------------------------------+ - Example 5: Parsing XML with nested arrays + Example 4: Counting distinct values with a specified relative standard deviation >>> from pyspark.sql import functions as sf - >>> df.select( - ... sf.schema_of_xml( - ... sf.lit('12') - ... ).alias("xml") - ... ).collect() - [Row(xml='STRUCT>>')] - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` + >>> spark.range(100000).agg( + ... sf.approx_count_distinct("id").alias('with_default_rsd'), + ... sf.approx_count_distinct("id", 0.1).alias('with_rsd_0.1') + ... ).show() + +----------------+------------+ + |with_default_rsd|with_rsd_0.1| + +----------------+------------+ + | 95546| 102065| + +----------------+------------+ """ from pyspark.sql.classic.column import _to_java_column - xml = _enum_to_value(xml) - if not isinstance(xml, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "xml", - "arg_type": type(xml).__name__, - }, - ) - - return _invoke_function("schema_of_xml", _to_java_column(lit(xml)), _options_to_str(options)) + if rsd is None: + return _invoke_function_over_columns("approx_count_distinct", col) + else: + return _invoke_function("approx_count_distinct", _to_java_column(col), _enum_to_value(rsd)) @_try_remote_functions -def to_xml(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: - """ - Converts a column containing a :class:`StructType` into a XML string. - Throws an exception, in the case of an unsupported type. +def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for + ``col1`` and ``col2``. - .. versionadded:: 4.0.0 + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a struct, array, map, or variant. - name of column containing a struct. - options: dict, optional - options to control converting. accepts the same options as the XML datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col1 : :class:`~pyspark.sql.Column` or column name + first column to calculate correlation. + A column that evaluates to a numeric. + col2 : :class:`~pyspark.sql.Column` or column name + second column to calculate correlation. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - a XML string converted from given :class:`StructType`. - Returns a column that evaluates to a string. + Pearson Correlation Coefficient of these two column values. Examples -------- - >>> from pyspark.sql import Row - >>> data = [(1, Row(age=2, name='Alice'))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(to_xml(df.value, {'rowTag':'person'}).alias("xml")).collect() - [Row(xml='\\n 2\\n Alice\\n')] + >>> from pyspark.sql import functions as sf + >>> a = range(20) + >>> b = [2 * x for x in range(20)] + >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) + >>> df.agg(sf.corr("a", df.b)).show() + +----------+ + |corr(a, b)| + +----------+ + | 1.0| + +----------+ + """ + return _invoke_function_over_columns("corr", col1, col2) + + +@_try_remote_functions +def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and + ``col2``. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + first column to calculate covariance. + A column that evaluates to a numeric. + col2 : :class:`~pyspark.sql.Column` or column name + second column to calculate covariance. + A column that evaluates to a numeric. + + Returns + ------- + :class:`~pyspark.sql.Column` + covariance of these two column values. See Also -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - from pyspark.sql.classic.column import _to_java_column + :meth:`pyspark.sql.functions.covar_samp` - return _invoke_function("to_xml", _to_java_column(col), _options_to_str(options)) + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> a = [1] * 10 + >>> b = [1] * 10 + >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) + >>> df.agg(sf.covar_pop("a", df.b)).show() + +---------------+ + |covar_pop(a, b)| + +---------------+ + | 0.0| + +---------------+ + """ + return _invoke_function_over_columns("covar_pop", col1, col2) @_try_remote_functions -def schema_of_csv(csv: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: - """ - CSV Function: Parses a CSV string and infers its schema in DDL format. +def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and + ``col2``. - .. versionadded:: 3.0.0 + .. versionadded:: 2.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - csv : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - A CSV string or a foldable string column containing a CSV string. - options : dict, optional - Options to control parsing. Accepts the same options as the CSV datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col1 : :class:`~pyspark.sql.Column` or column name + first column to calculate covariance. + A column that evaluates to a numeric. + col2 : :class:`~pyspark.sql.Column` or column name + second column to calculate covariance. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - A string representation of a :class:`StructType` parsed from the given CSV. - Returns a column that evaluates to a string. + sample covariance of these two column values. - Examples + See Also -------- - Example 1: Inferring the schema of a CSV string with different data types + :meth:`pyspark.sql.functions.covar_pop` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('1|a|true'), {'sep':'|'})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv(1|a|true) | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ + >>> a = [1] * 10 + >>> b = [1] * 10 + >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) + >>> df.agg(sf.covar_samp("a", df.b)).show() + +----------------+ + |covar_samp(a, b)| + +----------------+ + | 0.0| + +----------------+ + """ + return _invoke_function_over_columns("covar_samp", col1, col2) - Example 2: Inferring the schema of a CSV string with missing values - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('1||true'), {'sep':'|'})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv(1||true) | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ +@_try_remote_functions +def countDistinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``. - Example 3: Inferring the schema of a CSV string with a different delimiter + An alias of :func:`count_distinct`, and it is encouraged to use :func:`count_distinct` + directly. - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('1;a;true'), {'sep':';'})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv(1;a;true) | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ + .. versionadded:: 1.3.0 - Example 4: Inferring the schema of a CSV string with quoted fields + .. versionchanged:: 3.4.0 + Supports Spark Connect. + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('"1","a","true"'), {'sep':','})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv("1","a","true") | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column - - csv = _enum_to_value(csv) - if not isinstance(csv, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "csv", - "arg_type": type(csv).__name__, - }, - ) + >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) + >>> df.select(sf.count_distinct(df.value)).show() + +---------------------+ + |count(DISTINCT value)| + +---------------------+ + | 2| + +---------------------+ - return _invoke_function("schema_of_csv", _to_java_column(lit(csv)), _options_to_str(options)) + >>> df.select(sf.countDistinct(df.value)).show() + +---------------------+ + |count(DISTINCT value)| + +---------------------+ + | 2| + +---------------------+ + """ + return count_distinct(col, *cols) @_try_remote_functions -def to_csv(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: - """ - CSV Function: Converts a column containing a :class:`StructType` into a CSV string. - Throws an exception, in the case of an unsupported type. +def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: + """Returns a new :class:`Column` for distinct count of ``col`` or ``cols``. - .. versionadded:: 3.0.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a struct, array, map, or variant. - Name of column containing a struct. - options: dict, optional - Options to control converting. Accepts the same options as the CSV datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col : :class:`~pyspark.sql.Column` or column name + first column to compute on. + cols : :class:`~pyspark.sql.Column` or column name + other columns to compute on. Returns ------- :class:`~pyspark.sql.Column` - A CSV string converted from the given :class:`StructType`. - Returns a column that evaluates to a string. + distinct values of these two column values. - Examples + See Also -------- - Example 1: Converting a simple StructType to a CSV string - - >>> from pyspark.sql import Row, functions as sf - >>> data = [(1, Row(age=2, name='Alice'))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_csv(df.value)).show() - +-------------+ - |to_csv(value)| - +-------------+ - | 2,Alice| - +-------------+ + :meth:`pyspark.sql.functions.approx_count_distinct` - Example 2: Converting a complex StructType to a CSV string + Examples + -------- + Example 1: Counting distinct values of a single column - >>> from pyspark.sql import Row, functions as sf - >>> data = [(1, Row(age=2, name='Alice', scores=[100, 200, 300]))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_csv(df.value)).show(truncate=False) - +-------------------------+ - |to_csv(value) | - +-------------------------+ - |2,Alice,"[100, 200, 300]"| - +-------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) + >>> df.select(sf.count_distinct(df.value)).show() + +---------------------+ + |count(DISTINCT value)| + +---------------------+ + | 2| + +---------------------+ - Example 3: Converting a StructType with null values to a CSV string + Example 2: Counting distinct values of multiple columns - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import StructType, StructField, IntegerType, StringType - >>> data = [(1, Row(age=None, name='Alice'))] - >>> schema = StructType([ - ... StructField("key", IntegerType(), True), - ... StructField("value", StructType([ - ... StructField("age", IntegerType(), True), - ... StructField("name", StringType(), True) - ... ]), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.to_csv(df.value)).show() - +-------------+ - |to_csv(value)| - +-------------+ - | ,Alice| - +-------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) + >>> df.select(sf.count_distinct(df.value1, df.value2)).show() + +------------------------------+ + |count(DISTINCT value1, value2)| + +------------------------------+ + | 2| + +------------------------------+ - Example 4: Converting a StructType with different data types to a CSV string + Example 3: Counting distinct values with column names as strings - >>> from pyspark.sql import Row, functions as sf - >>> data = [(1, Row(age=2, name='Alice', isStudent=True))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_csv(df.value)).show() - +-------------+ - |to_csv(value)| - +-------------+ - | 2,Alice,true| - +-------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) + >>> df.select(sf.count_distinct("value1", "value2")).show() + +------------------------------+ + |count(DISTINCT value1, value2)| + +------------------------------+ + | 2| + +------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column + from pyspark.sql.classic.column import _to_java_column, _to_seq - return _invoke_function("to_csv", _to_java_column(col), _options_to_str(options)) + sc = _get_active_spark_context() + return _invoke_function( + "count_distinct", _to_java_column(col), _to_seq(sc, cols, _to_java_column) + ) @_try_remote_functions -def size(col: "ColumnOrName") -> Column: - """ - Collection function: returns the length of the array or map stored in the column. +def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: + """Aggregate function: returns the first value in a group. - .. versionadded:: 1.5.0 + The function by default returns the first values it sees. It will return the first non-null + value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Notes + ----- + The function is non-deterministic because its results depends on the order of the + rows which may be non-deterministic after a shuffle. + Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array or map. + col : :class:`~pyspark.sql.Column` or column name + column to fetch first value for. + A column of any type. + ignorenulls : bool + if first value is null then look for first non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - length of the array/map. - Returns a column that evaluates to an integer. + first value of the group. Examples -------- - >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data']) - >>> df.select(size(df.data)).collect() - [Row(size(data)=3), Row(size(data)=1), Row(size(data)=0)] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) + >>> df = df.orderBy(df.age) + >>> df.groupby("name").agg(sf.first("age")).orderBy("name").show() + +-----+----------+ + | name|first(age)| + +-----+----------+ + |Alice| NULL| + | Bob| 5| + +-----+----------+ + + To ignore any null values, set ``ignorenulls`` to `True` + + >>> df.groupby("name").agg(sf.first("age", ignorenulls=True)).orderBy("name").show() + +-----+----------+ + | name|first(age)| + +-----+----------+ + |Alice| 2| + | Bob| 5| + +-----+----------+ """ - return _invoke_function_over_columns("size", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("first", _to_java_column(col), _enum_to_value(ignorenulls)) @_try_remote_functions -def array_min(col: "ColumnOrName") -> Column: +def grouping(col: "ColumnOrName") -> Column: """ - Array function: returns the minimum value of the array. + Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated + or not, returns 1 for aggregated or 0 for not aggregated in the result set. - .. versionadded:: 2.4.0 + .. versionadded:: 2.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the array. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + column to check if it's aggregated. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains the minimum value of each array. - Returns a column of the element type of the input array. - - See Also - -------- - :meth:`pyspark.sql.functions.array_max` - :meth:`pyspark.sql.functions.array_sort` - :meth:`pyspark.sql.functions.sort_array` + returns 1 for aggregated or 0 for not aggregated in the result set. Examples -------- - Example 1: Basic usage with integer array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | 1| - | -1| - +---------------+ - - Example 2: Usage with string array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | apple| - +---------------+ - - Example 3: Usage with mixed type array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | 1| - +---------------+ - - Example 4: Usage with array of arrays - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | [2, 1]| - +---------------+ - - Example 5: Usage with empty array - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | NULL| - +---------------+ + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) + >>> df.cube("name").agg(sf.grouping("name"), sf.sum("age")).orderBy("name").show() + +-----+--------------+--------+ + | name|grouping(name)|sum(age)| + +-----+--------------+--------+ + | NULL| 1| 7| + |Alice| 0| 2| + | Bob| 0| 5| + +-----+--------------+--------+ """ - return _invoke_function_over_columns("array_min", col) + return _invoke_function_over_columns("grouping", col) @_try_remote_functions -def array_max(col: "ColumnOrName") -> Column: +def grouping_id(*cols: "ColumnOrName") -> Column: """ - Array function: returns the maximum value of the array. + Aggregate function: returns the level of grouping, equals to - .. versionadded:: 2.4.0 + (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn) + + .. versionadded:: 2.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Notes + ----- + The list of columns should match with grouping columns exactly, or empty (means all + the grouping columns). + Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the array. - A column that evaluates to an array. + cols : :class:`~pyspark.sql.Column` or column name + columns to check for. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains the maximum value of each array. - Returns a column of the element type of the input array. - - See Also - -------- - :meth:`pyspark.sql.functions.array_min` - :meth:`pyspark.sql.functions.array_sort` - :meth:`pyspark.sql.functions.sort_array` + returns level of the grouping it relates to. Examples -------- - Example 1: Basic usage with integer array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | 3| - | 10| - +---------------+ - - Example 2: Usage with string array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | cherry| - +---------------+ - - Example 3: Usage with mixed type array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | cherry| - +---------------+ - - Example 4: Usage with array of arrays - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | [3, 4]| - +---------------+ - - Example 5: Usage with empty array - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | NULL| - +---------------+ + >>> df = spark.createDataFrame( + ... [(1, "a", "a"), (3, "a", "a"), (4, "b", "c")], ["c1", "c2", "c3"]) + >>> df.cube("c2", "c3").agg(sf.grouping_id(), sf.sum("c1")).orderBy("c2", "c3").show() + +----+----+-------------+-------+ + | c2| c3|grouping_id()|sum(c1)| + +----+----+-------------+-------+ + |NULL|NULL| 3| 8| + |NULL| a| 2| 4| + |NULL| c| 2| 4| + | a|NULL| 1| 4| + | a| a| 0| 4| + | b|NULL| 1| 4| + | b| c| 0| 4| + +----+----+-------------+-------+ """ - return _invoke_function_over_columns("array_max", col) + return _invoke_function_over_seq_of_columns("grouping_id", cols) @_try_remote_functions -def array_size(col: "ColumnOrName") -> Column: +def count_min_sketch( + col: "ColumnOrName", + eps: Union[Column, float], + confidence: Union[Column, float], + seed: Optional[Union[Column, int]] = None, +) -> Column: """ - Array function: returns the total number of elements in the array. - The function returns null for null input. + Returns a count-min sketch of a column with the given esp, confidence and seed. + The result is an array of bytes, which can be deserialized to a `CountMinSketch` before usage. + Count-min sketch is a probabilistic data structure used for cardinality estimation + using sub-linear space. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the array. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + eps : :class:`~pyspark.sql.Column` or float + relative error, must be positive + + .. versionchanged:: 4.0.0 + `eps` now accepts float value. + + confidence : :class:`~pyspark.sql.Column` or float + confidence, must be positive and less than 1.0 + + .. versionchanged:: 4.0.0 + `confidence` now accepts float value. + + seed : :class:`~pyspark.sql.Column` or int, optional + random seed + + .. versionchanged:: 4.0.0 + `seed` now accepts int value. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains the size of each array. - Returns a column that evaluates to an integer. - - See Also - -------- - :meth:`pyspark.sql.functions.cardinality` - :meth:`pyspark.sql.functions.size` + count-min sketch of the column Examples -------- - Example 1: Basic usage with integer array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],), (None,)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 3| - | NULL| - +----------------+ - - Example 2: Usage with string array + Example 1: Using columns as arguments >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 3| - +----------------+ + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch(sf.col("id"), sf.lit(3.0), sf.lit(0.1), sf.lit(1))) + ... ).show(truncate=False) + +------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 3.0, 0.1, 1)) | + +------------------------------------------------------------------------+ + |0000000100000000000000640000000100000001000000005D8D6AB90000000000000064| + +------------------------------------------------------------------------+ - Example 3: Usage with mixed type array + Example 2: Using numbers as arguments >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 3| - +----------------+ + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch("id", 1.0, 0.3, 2)) + ... ).show(truncate=False) + +----------------------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 1.0, 0.3, 2)) | + +----------------------------------------------------------------------------------------+ + |0000000100000000000000640000000100000002000000005D96391C00000000000000320000000000000032| + +----------------------------------------------------------------------------------------+ - Example 4: Usage with array of arrays + Example 3: Using a long seed >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 2| - +----------------+ + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.2, 1111111111111111111)) + ... ).show(truncate=False) + +----------------------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 1.5, 0.2, 1111111111111111111)) | + +----------------------------------------------------------------------------------------+ + |00000001000000000000006400000001000000020000000044078BA100000000000000320000000000000032| + +----------------------------------------------------------------------------------------+ - Example 5: Usage with empty array + Example 4: Using a random seed >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 0| - +----------------+ + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.6)) + ... ).show(truncate=False) # doctest: +SKIP + +----------------------------------------------------------------------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 1.5, 0.6, 2120704260)) | + +----------------------------------------------------------------------------------------------------------------------------------------+ + |0000000100000000000000640000000200000002000000005ADECCEE00000000153EBE090000000000000033000000000000003100000000000000320000000000000032| + +----------------------------------------------------------------------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("array_size", col) + _eps = lit(eps) + _conf = lit(confidence) + if seed is None: + return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf) + else: + return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf, lit(seed)) @_try_remote_functions -def cardinality(col: "ColumnOrName") -> Column: - """ - Collection function: returns the length of the array or map stored in the column. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to an array or map. - - Returns - ------- - :class:`~pyspark.sql.Column` - length of the array/map. - Returns a column that evaluates to an integer. - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [([1, 2, 3],),([1],),([],)], ['data'] - ... ).select(sf.cardinality("data")).show() - +-----------------+ - |cardinality(data)| - +-----------------+ - | 3| - | 1| - | 0| - +-----------------+ - """ - return _invoke_function_over_columns("cardinality", col) - +def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: + """Aggregate function: returns the last value in a group. -@_try_remote_functions -def sort_array(col: "ColumnOrName", asc: bool = True) -> Column: - """ - Array function: Sorts the input array in ascending or descending order according - to the natural ordering of the array elements. Null elements will be placed at the beginning - of the returned array in ascending order or at the end of the returned array in descending - order. + The function by default returns the last values it sees. It will return the last non-null + value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - .. versionadded:: 1.5.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Notes + ----- + The function is non-deterministic because its results depends on the order of the + rows which may be non-deterministic after a shuffle. + Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of the column or expression. - A column that evaluates to an array. - asc : bool, optional - Whether to sort in ascending or descending order. If `asc` is True (default), - then the sorting is in ascending order. If False, then in descending order. + col : :class:`~pyspark.sql.Column` or column name + column to fetch last value for. + A column of any type. + ignorenulls : bool + if last value is null then look for non-null value. ``False`` by default. A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - Sorted array. - Returns a column that evaluates to an array. + last value of the group. Examples -------- - Example 1: Sorting an array in ascending order - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | [NULL, 1, 2, 3]| - +----------------------+ - - Example 2: Sorting an array in descending order - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) - >>> df.select(sf.sort_array(df.data, asc=False)).show() - +-----------------------+ - |sort_array(data, false)| - +-----------------------+ - | [3, 2, 1, NULL]| - +-----------------------+ - - Example 3: Sorting an array with a single element - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([([1],)], ['data']) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | [1]| - +----------------------+ - - Example 4: Sorting an empty array - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | []| - +----------------------+ + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) + >>> df = df.orderBy(df.age.desc()) + >>> df.groupby("name").agg(sf.last("age")).orderBy("name").show() + +-----+---------+ + | name|last(age)| + +-----+---------+ + |Alice| NULL| + | Bob| 5| + +-----+---------+ - Example 5: Sorting an array with null values + To ignore any null values, set ``ignorenulls`` to `True` - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) - >>> df = spark.createDataFrame([([None, None, None],)], schema=schema) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | [NULL, NULL, NULL]| - +----------------------+ + >>> df.groupby("name").agg(sf.last("age", ignorenulls=True)).orderBy("name").show() + +-----+---------+ + | name|last(age)| + +-----+---------+ + |Alice| 2| + | Bob| 5| + +-----+---------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function("sort_array", _to_java_column(col), _enum_to_value(asc)) + return _invoke_function("last", _to_java_column(col), _enum_to_value(ignorenulls)) @_try_remote_functions -def array_sort( - col: "ColumnOrName", comparator: Optional[Callable[[Column, Column], Column]] = None +def percentile( + col: "ColumnOrName", + percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], + frequency: Union[Column, int] = 1, ) -> Column: - """ - Collection function: sorts the input array in ascending order. The elements of the input array - must be orderable. Null elements will be placed at the end of the returned array. - - .. versionadded:: 2.4.0 - - .. versionchanged:: 3.4.0 - Can take a `comparator` function. + """Returns the exact percentile(s) of numeric column `expr` at the given percentage(s) + with value range in [0.0, 1.0]. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - comparator : callable, optional - A binary ``(Column, Column) -> Column: ...``. - The comparator will take two - arguments representing two elements of the array. It returns a negative integer, 0, or a - positive integer as the first element is less than, equal to, or greater than the second - element. If the comparator function returns null, the function will fail and raise an error. + col : :class:`~pyspark.sql.Column` or column name + percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats + percentage in decimal (must be between 0.0 and 1.0). + frequency : :class:`~pyspark.sql.Column` or int is a positive numeric literal which + controls frequency. Returns ------- :class:`~pyspark.sql.Column` - sorted array. - Returns a column that evaluates to an array. + the exact `percentile` of the numeric column. See Also -------- - :meth:`pyspark.sql.functions.sort_array` + :meth:`pyspark.sql.functions.median` + :meth:`pyspark.sql.functions.approx_percentile` + :meth:`pyspark.sql.functions.percentile_approx` Examples -------- - >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data']) - >>> df.select(array_sort(df.data).alias('r')).collect() - [Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])] - >>> df = spark.createDataFrame([(["foo", "foobar", None, "bar"],),(["foo"],),([],)], ['data']) - >>> df.select(array_sort( - ... "data", - ... lambda x, y: when(x.isNull() | y.isNull(), lit(0)).otherwise(length(y) - length(x)) - ... ).alias("r")).collect() - [Row(r=['foobar', 'foo', None, 'bar']), Row(r=['foo']), Row(r=[])] - """ - if comparator is None: - return _invoke_function_over_columns("array_sort", col) - else: - return _invoke_higher_order_function("array_sort", [col], [comparator]) - - -@_try_remote_functions -def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> Column: - """ - Array function: Generates a random permutation of the given array. + >>> from pyspark.sql import functions as sf + >>> key = (sf.col("id") % 3).alias("key") + >>> value = (sf.randn(42) + key * 10).alias("value") + >>> df = spark.range(0, 1000, 1, 1).select(key, value) + >>> df.select( + ... sf.percentile("value", [0.25, 0.5, 0.75], sf.lit(1)) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |percentile(value, array(0.25, 0.5, 0.75), 1) | + +--------------------------------------------------------+ + |[0.7441991494121..., 9.9900713756..., 19.33740203080...]| + +--------------------------------------------------------+ - .. versionadded:: 2.4.0 + >>> df.groupBy("key").agg( + ... sf.percentile("value", sf.lit(0.5), sf.lit(1)) + ... ).sort("key").show() + +---+-------------------------+ + |key|percentile(value, 0.5, 1)| + +---+-------------------------+ + | 0| -0.03449962216667901| + | 1| 9.990389751837329| + | 2| 19.967859769284075| + +---+-------------------------+ + """ + percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) + return _invoke_function_over_columns("percentile", col, percentage, lit(frequency)) + + +@_try_remote_functions +def percentile_approx( + col: "ColumnOrName", + percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], + accuracy: Union[Column, int] = 10000, +) -> Column: + """Returns the approximate `percentile` of the numeric column `col` which is the smallest value + in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` + of `col` values is less than the value or equal to that value. + + + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or expression to be shuffled. - A column that evaluates to an array. - seed : :class:`~pyspark.sql.Column` or int, optional - Seed value for the random generator. - A column that evaluates to an integer or long. Must be a constant. - - .. versionadded:: 4.0.0 + col : :class:`~pyspark.sql.Column` or column name + input column. + percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats + percentage in decimal (must be between 0.0 and 1.0). + When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. + In this case, returns the approximate percentile array of column col + at the given percentage array. + accuracy : :class:`~pyspark.sql.Column` or int + is a positive numeric literal which controls approximation accuracy + at the cost of memory. Higher value of accuracy yields better accuracy, + 1.0/accuracy is the relative error of the approximation. (default: 10000). Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an array of elements in random order. - Returns a column that evaluates to an array. + approximate `percentile` of the numeric column. - Notes - ----- - The `shuffle` function is non-deterministic, meaning the order of the output array - can be different for each execution. + See Also + -------- + :meth:`pyspark.sql.functions.median` + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.approx_percentile` Examples -------- - Example 1: Shuffling a simple array + >>> from pyspark.sql import functions as sf + >>> key = (sf.col("id") % 3).alias("key") + >>> value = (sf.randn(42) + key * 10).alias("value") + >>> df = spark.range(0, 1000, 1, 1).select(key, value) + >>> df.select( + ... sf.percentile_approx("value", [0.25, 0.5, 0.75], 1000000) + ... ).show(truncate=False) + +----------------------------------------------------------+ + |percentile_approx(value, array(0.25, 0.5, 0.75), 1000000) | + +----------------------------------------------------------+ + |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| + +----------------------------------------------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 20, 3, 5) AS data") - >>> df.select("*", sf.shuffle(df.data, sf.lit(123))).show() # doctest: +SKIP - +-------------+-------------+ - | data|shuffle(data)| - +-------------+-------------+ - |[1, 20, 3, 5]|[5, 1, 20, 3]| - +-------------+-------------+ + >>> df.groupBy("key").agg( + ... sf.percentile_approx("value", sf.lit(0.5), sf.lit(1000000)) + ... ).sort("key").show() + +---+--------------------------------------+ + |key|percentile_approx(value, 0.5, 1000000)| + +---+--------------------------------------+ + | 0| -0.03519435193070...| + | 1| 9.990389751837...| + | 2| 19.967859769284...| + +---+--------------------------------------+ + """ + percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) + return _invoke_function_over_columns("percentile_approx", col, percentage, lit(accuracy)) - Example 2: Shuffling an array with null values - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 20, NULL, 5) AS data") - >>> df.select("*", sf.shuffle(sf.col("data"), 234)).show() # doctest: +SKIP - +----------------+----------------+ - | data| shuffle(data)| - +----------------+----------------+ - |[1, 20, NULL, 5]|[NULL, 5, 20, 1]| - +----------------+----------------+ +@_try_remote_functions +def approx_percentile( + col: "ColumnOrName", + percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], + accuracy: Union[Column, int] = 10000, +) -> Column: + """Returns the approximate `percentile` of the numeric column `col` which is the smallest value + in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` + of `col` values is less than the value or equal to that value. - Example 3: Shuffling an array with duplicate values + .. versionadded:: 3.5.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") - >>> df.select("*", sf.shuffle("data", 345)).show() # doctest: +SKIP - +------------------+------------------+ - | data| shuffle(data)| - +------------------+------------------+ - |[1, 2, 2, 3, 3, 3]|[2, 3, 3, 1, 2, 3]| - +------------------+------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input column. + percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats + percentage in decimal (must be between 0.0 and 1.0). + When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. + In this case, returns the approximate percentile array of column col + at the given percentage array. + accuracy : :class:`~pyspark.sql.Column` or int + is a positive numeric literal which controls approximation accuracy + at the cost of memory. Higher value of accuracy yields better accuracy, + 1.0/accuracy is the relative error of the approximation. (default: 10000). - Example 4: Shuffling an array with random seed + Returns + ------- + :class:`~pyspark.sql.Column` + approximate `percentile` of the numeric column. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") - >>> df.select("*", sf.shuffle("data")).show() # doctest: +SKIP - +------------------+------------------+ - | data| shuffle(data)| - +------------------+------------------+ - |[1, 2, 2, 3, 3, 3]|[3, 3, 2, 3, 2, 1]| - +------------------+------------------+ - """ - if seed is not None: - return _invoke_function_over_columns("shuffle", col, lit(seed)) - else: - return _invoke_function_over_columns("shuffle", col) + See Also + -------- + :meth:`pyspark.sql.functions.median` + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.percentile_approx` + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> key = (sf.col("id") % 3).alias("key") + >>> value = (sf.randn(42) + key * 10).alias("value") + >>> df = spark.range(0, 1000, 1, 1).select(key, value) + >>> df.select( + ... sf.approx_percentile("value", [0.25, 0.5, 0.75], 1000000) + ... ).show(truncate=False) + +----------------------------------------------------------+ + |approx_percentile(value, array(0.25, 0.5, 0.75), 1000000) | + +----------------------------------------------------------+ + |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| + +----------------------------------------------------------+ -@_try_remote_functions -def reverse(col: "ColumnOrName") -> Column: + >>> df.groupBy("key").agg( + ... sf.approx_percentile("value", sf.lit(0.5), sf.lit(1000000)) + ... ).sort("key").show() + +---+--------------------------------------+ + |key|approx_percentile(value, 0.5, 1000000)| + +---+--------------------------------------+ + | 0| -0.03519435193070...| + | 1| 9.990389751837...| + | 2| 19.967859769284...| + +---+--------------------------------------+ """ - Collection function: returns a reversed string, a binary value with bytes in reverse order, - or an array with elements in reverse order. + percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) + return _invoke_function_over_columns("approx_percentile", col, percentage, lit(accuracy)) - .. versionadded:: 1.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def any_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: + """Returns some value of `col` for a group of rows. - .. versionchanged:: 4.2.0 - Added support for binary type. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the element to be reversed. - A column that evaluates to a string, binary, or array. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column of any type. + ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional + if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a reversed string, a binary value with bytes in reverse order, - or an array with elements in reverse order. - Returns a column of the same type as the input. + some value of `col` for a group of rows. Examples -------- - Example 1: Reverse a string - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark SQL',)], ['data']) - >>> df.select(sf.reverse(df.data)).show() - +-------------+ - |reverse(data)| - +-------------+ - | LQS krapS| - +-------------+ - - Example 2: Reverse an array - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],) ,([1],) ,([],)], ['data']) - >>> df.select(sf.reverse(df.data)).show() - +-------------+ - |reverse(data)| - +-------------+ - | [3, 1, 2]| - | [1]| - | []| - +-------------+ - - Example 3: Reverse binary data + >>> df = spark.createDataFrame( + ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.select(sf.any_value('c1'), sf.any_value('c2')).show() + +-------------+-------------+ + |any_value(c1)|any_value(c2)| + +-------------+-------------+ + | NULL| 1| + +-------------+-------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytearray(b"\\xCA\\xFE"),)], "data: binary") - >>> df.select(sf.hex(sf.reverse(df.data))).show() - +------------------+ - |hex(reverse(data))| - +------------------+ - | FECA| - +------------------+ + >>> df.select(sf.any_value('c1', True), sf.any_value('c2', True)).show() + +-------------+-------------+ + |any_value(c1)|any_value(c2)| + +-------------+-------------+ + | a| 1| + +-------------+-------------+ """ - return _invoke_function_over_columns("reverse", col) + if ignoreNulls is None: + return _invoke_function_over_columns("any_value", col) + else: + ignoreNulls = _enum_to_value(ignoreNulls) + ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls + return _invoke_function_over_columns("any_value", col, ignoreNulls) @_try_remote_functions -def flatten(col: "ColumnOrName") -> Column: - """ - Array function: creates a single array from an array of arrays. - If a structure of nested arrays is deeper than two levels, - only one level of nesting is removed. - - .. versionadded:: 2.4.0 +def first_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: + """Returns the first value of `col` for a group of rows. It will return the first non-null + value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or expression to be flattened. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column of any type. + ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional + if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains the flattened array. + some value of `col` for a group of rows. - Examples + See Also -------- - Example 1: Flattening a simple nested array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[1, 2, 3], [4, 5], [6]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show() - +------------------+ - | flatten(data)| - +------------------+ - |[1, 2, 3, 4, 5, 6]| - +------------------+ - - Example 2: Flattening an array with null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([None, [4, 5]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show() - +-------------+ - |flatten(data)| - +-------------+ - | NULL| - +-------------+ - - Example 3: Flattening an array with more than two levels of nesting - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[[1, 2], [3, 4]], [[5, 6], [7, 8]]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show(truncate=False) - +--------------------------------+ - |flatten(data) | - +--------------------------------+ - |[[1, 2], [3, 4], [5, 6], [7, 8]]| - +--------------------------------+ + :meth:`pyspark.sql.functions.last_value` + :meth:`pyspark.sql.functions.nth_value` - Example 4: Flattening an array with mixed types + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] + ... ).select(sf.first_value('a'), sf.first_value('b')).show() + +--------------+--------------+ + |first_value(a)|first_value(b)| + +--------------+--------------+ + | NULL| 1| + +--------------+--------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([['a', 'b', 'c'], [1, 2, 3]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show() - +------------------+ - | flatten(data)| - +------------------+ - |[a, b, c, 1, 2, 3]| - +------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] + ... ).select(sf.first_value('a', True), sf.first_value('b', True)).show() + +--------------+--------------+ + |first_value(a)|first_value(b)| + +--------------+--------------+ + | a| 1| + +--------------+--------------+ """ - return _invoke_function_over_columns("flatten", col) + if ignoreNulls is None: + return _invoke_function_over_columns("first_value", col) + else: + ignoreNulls = _enum_to_value(ignoreNulls) + ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls + return _invoke_function_over_columns("first_value", col, ignoreNulls) @_try_remote_functions -def map_contains_key(col: "ColumnOrName", value: Any) -> Column: - """ - Map function: Returns true if the map contains the key. - - .. versionadded:: 3.4.0 +def last_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: + """Returns the last value of `col` for a group of rows. It will return the last non-null + value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the map. - value : - A literal value, or a :class:`~pyspark.sql.Column` expression. - - .. versionchanged:: 4.0.0 - `value` now also accepts a Column type. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column of any type. + ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional + if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - True if key is in the map and False otherwise. + some value of `col` for a group of rows. - Examples + See Also -------- - Example 1: The key is in the map - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.map_contains_key("data", 1)).show() - +-------------------------+ - |map_contains_key(data, 1)| - +-------------------------+ - | true| - +-------------------------+ - - Example 2: The key is not in the map - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.map_contains_key("data", -1)).show() - +--------------------------+ - |map_contains_key(data, -1)| - +--------------------------+ - | false| - +--------------------------+ + :meth:`pyspark.sql.functions.first_value` + :meth:`pyspark.sql.functions.nth_value` - Example 3: Check for key using a column + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] + ... ).select(sf.last_value('a'), sf.last_value('b')).show() + +-------------+-------------+ + |last_value(a)|last_value(b)| + +-------------+-------------+ + | NULL| 2| + +-------------+-------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data, 1 as key") - >>> df.select(sf.map_contains_key("data", sf.col("key"))).show() - +---------------------------+ - |map_contains_key(data, key)| - +---------------------------+ - | true| - +---------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] + ... ).select(sf.last_value('a', True), sf.last_value('b', True)).show() + +-------------+-------------+ + |last_value(a)|last_value(b)| + +-------------+-------------+ + | b| 2| + +-------------+-------------+ """ - return _invoke_function_over_columns("map_contains_key", col, lit(value)) + if ignoreNulls is None: + return _invoke_function_over_columns("last_value", col) + else: + ignoreNulls = _enum_to_value(ignoreNulls) + ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls + return _invoke_function_over_columns("last_value", col, ignoreNulls) @_try_remote_functions -def map_keys(col: "ColumnOrName") -> Column: +def count_if(col: "ColumnOrName") -> Column: """ - Map function: Returns an unordered array containing the keys of the map. - - .. versionadded:: 2.3.0 + Aggregate function: Returns the number of `TRUE` values for the `col`. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of column or expression + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - Keys of the map as an array. + the number of `TRUE` values for the `col`. + + See Also + -------- + :meth:`pyspark.sql.functions.count` Examples -------- - Example 1: Extracting keys from a simple map + Example 1: Counting the number of even numbers in a numeric column >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_keys("data"))).show() - +--------------------------------+ - |sort_array(map_keys(data), true)| - +--------------------------------+ - | [1, 2]| - +--------------------------------+ + >>> df = spark.createDataFrame([("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.select(sf.count_if(sf.col('c2') % 2 == 0)).show() + +------------------------+ + |count_if(((c2 % 2) = 0))| + +------------------------+ + | 3| + +------------------------+ - Example 2: Extracting keys from a map with complex keys + Example 2: Counting the number of rows where a string column starts with a certain letter >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(array(1, 2), 'a', array(3, 4), 'b') as data") - >>> df.select(sf.sort_array(sf.map_keys("data"))).show() - +--------------------------------+ - |sort_array(map_keys(data), true)| - +--------------------------------+ - | [[1, 2], [3, 4]]| - +--------------------------------+ + >>> df = spark.createDataFrame( + ... [("apple",), ("banana",), ("cherry",), ("apple",), ("banana",)], ["fruit"]) + >>> df.select(sf.count_if(sf.col('fruit').startswith('a'))).show() + +------------------------------+ + |count_if(startswith(fruit, a))| + +------------------------------+ + | 2| + +------------------------------+ - Example 3: Extracting keys from a map with duplicate keys + Example 3: Counting the number of rows where a numeric column is greater than a certain value >>> from pyspark.sql import functions as sf - >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") - >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") - >>> df.select(sf.map_keys("data")).show() - +--------------+ - |map_keys(data)| - +--------------+ - | [1]| - +--------------+ - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) + >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,), (5,)], ["num"]) + >>> df.select(sf.count_if(sf.col('num') > 3)).show() + +-------------------+ + |count_if((num > 3))| + +-------------------+ + | 2| + +-------------------+ - Example 4: Extracting keys from an empty map + Example 4: Counting the number of rows where a boolean column is True >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map() as data") - >>> df.select(sf.map_keys("data")).show() - +--------------+ - |map_keys(data)| - +--------------+ - | []| - +--------------+ + >>> df = spark.createDataFrame([(True,), (False,), (True,), (False,), (True,)], ["b"]) + >>> df.select(sf.count('b'), sf.count_if('b')).show() + +--------+-----------+ + |count(b)|count_if(b)| + +--------+-----------+ + | 5| 3| + +--------+-----------+ """ - return _invoke_function_over_columns("map_keys", col) + return _invoke_function_over_columns("count_if", col) @_try_remote_functions -def map_values(col: "ColumnOrName") -> Column: - """ - Map function: Returns an unordered array containing the values of the map. - - .. versionadded:: 2.3.0 +def histogram_numeric(col: "ColumnOrName", nBins: Column) -> Column: + """Computes a histogram on numeric 'col' using nb bins. + The return value is an array of (x,y) pairs representing the centers of the + histogram's bins. As the value of 'nb' is increased, the histogram approximation + gets finer-grained, but may yield artifacts around outliers. In practice, 20-40 + histogram bins appear to work well, with more bins being required for skewed or + smaller datasets. Note that this function creates a histogram with non-uniform + bin widths. It offers no guarantees in terms of the mean-squared-error of the + histogram, but in practice is comparable to the histograms produced by the R/S-Plus + statistical computing packages. Note: the output type of the 'x' field in the return value is + propagated from the input value consumed in the aggregate function. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of column or expression + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + nBins : :class:`~pyspark.sql.Column` + number of Histogram columns. Returns ------- :class:`~pyspark.sql.Column` - Values of the map as an array. + a histogram on numeric 'col' using nb bins. Examples -------- - Example 1: Extracting values from a simple map - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_values("data"))).show() - +----------------------------------+ - |sort_array(map_values(data), true)| - +----------------------------------+ - | [a, b]| - +----------------------------------+ - - Example 2: Extracting values from a map with complex values - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, array('a', 'b'), 2, array('c', 'd')) as data") - >>> df.select(sf.sort_array(sf.map_values("data"))).show() - +----------------------------------+ - |sort_array(map_values(data), true)| - +----------------------------------+ - | [[a, b], [c, d]]| - +----------------------------------+ - - Example 3: Extracting values from a map with null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, null, 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_values("data"))).show() - +----------------------------------+ - |sort_array(map_values(data), true)| - +----------------------------------+ - | [NULL, b]| - +----------------------------------+ - - Example 4: Extracting values from a map with duplicate values - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'a') as data") - >>> df.select(sf.map_values("data")).show() - +----------------+ - |map_values(data)| - +----------------+ - | [a, a]| - +----------------+ - - Example 5: Extracting values from an empty map - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map() as data") - >>> df.select(sf.map_values("data")).show() - +----------------+ - |map_values(data)| - +----------------+ - | []| - +----------------+ + >>> df = spark.range(100, numPartitions=1) + >>> df.select(sf.histogram_numeric('id', sf.lit(5))).show(truncate=False) + +-----------------------------------------------------------+ + |histogram_numeric(id, 5) | + +-----------------------------------------------------------+ + |[{11, 25.0}, {36, 24.0}, {59, 23.0}, {84, 25.0}, {98, 3.0}]| + +-----------------------------------------------------------+ """ - return _invoke_function_over_columns("map_values", col) + return _invoke_function_over_columns("histogram_numeric", col, nBins) @_try_remote_functions -def map_entries(col: "ColumnOrName") -> Column: +def hll_sketch_agg( + col: "ColumnOrName", + lgConfigK: Optional[Union[int, Column]] = None, +) -> Column: """ - Map function: Returns an unordered array of all entries in the given map. - - .. versionadded:: 3.0.0 + Aggregate function: returns the updatable binary representation of the Datasketches + HllSketch configured with lgConfigK arg. - .. versionchanged:: 3.4.0 - Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of column or expression + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an integer, long, string, or binary. + lgConfigK : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of K, where K is the number of buckets or slots for the HllSketch. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - An array of key value pairs as a struct type + The binary representation of the HllSketch. - Examples + See Also -------- - Example 1: Extracting entries from a simple map - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_entries("data"))).show() - +-----------------------------------+ - |sort_array(map_entries(data), true)| - +-----------------------------------+ - | [{1, a}, {2, b}]| - +-----------------------------------+ - - Example 2: Extracting entries from a map with complex keys and values - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(array(1, 2), array('a', 'b'), " - ... "array(3, 4), array('c', 'd')) as data") - >>> df.select(sf.sort_array(sf.map_entries("data"))).show(truncate=False) - +------------------------------------+ - |sort_array(map_entries(data), true) | - +------------------------------------+ - |[{[1, 2], [a, b]}, {[3, 4], [c, d]}]| - +------------------------------------+ - - Example 3: Extracting entries from a map with duplicate keys + :meth:`pyspark.sql.functions.hll_union` + :meth:`pyspark.sql.functions.hll_union_agg` + :meth:`pyspark.sql.functions.hll_sketch_estimate` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") - >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") - >>> df.select(sf.map_entries("data")).show() - +-----------------+ - |map_entries(data)| - +-----------------+ - | [{1, b}]| - +-----------------+ - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) - - Example 4: Extracting entries from an empty map + >>> df = spark.createDataFrame([1,2,2,3], "INT") + >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() + +----------------------------------------------+ + |hll_sketch_estimate(hll_sketch_agg(value, 12))| + +----------------------------------------------+ + | 3| + +----------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map() as data") - >>> df.select(sf.map_entries("data")).show() - +-----------------+ - |map_entries(data)| - +-----------------+ - | []| - +-----------------+ + >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value", 12))).show() + +----------------------------------------------+ + |hll_sketch_estimate(hll_sketch_agg(value, 12))| + +----------------------------------------------+ + | 3| + +----------------------------------------------+ """ - return _invoke_function_over_columns("map_entries", col) + if lgConfigK is None: + return _invoke_function_over_columns("hll_sketch_agg", col) + else: + return _invoke_function_over_columns("hll_sketch_agg", col, lit(lgConfigK)) @_try_remote_functions -def map_from_entries(col: "ColumnOrName") -> Column: +def hll_union_agg( + col: "ColumnOrName", + allowDifferentLgConfigK: Optional[Union[bool, Column]] = None, +) -> Column: """ - Map function: Transforms an array of key-value pair entries (structs with two fields) - into a map. The first field of each entry is used as the key and the second field - as the value in the resulting map column - - .. versionadded:: 2.4.0 + Aggregate function: returns the updatable binary representation of the Datasketches + HllSketch, generated by merging previously created Datasketches HllSketch instances + via a Datasketches Union instance. Throws an exception if sketches have different + lgConfigK values and allowDifferentLgConfigK is unset or set to false. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of column or expression + col : :class:`~pyspark.sql.Column` or column name + allowDifferentLgConfigK : :class:`~pyspark.sql.Column` or bool, optional + Allow sketches with different lgConfigK values to be merged (defaults to false). Returns ------- :class:`~pyspark.sql.Column` - A map created from the given array of entries. + The binary representation of the merged HllSketch. - Examples + See Also -------- - Example 1: Basic usage of map_from_entries - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array(struct(1, 'a'), struct(2, 'b')) as data") - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {1 -> a, 2 -> b}| - +----------------------+ - - Example 2: map_from_entries with null values + :meth:`pyspark.sql.functions.hll_union` + :meth:`pyspark.sql.functions.hll_sketch_agg` + :meth:`pyspark.sql.functions.hll_sketch_estimate` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array(struct(1, null), struct(2, 'b')) as data") - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {1 -> NULL, 2 -> b}| - +----------------------+ - - Example 3: map_from_entries with a DataFrame - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([([Row(1, "a"), Row(2, "b")],), ([Row(3, "c")],)], ['data']) - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {1 -> a, 2 -> b}| - | {3 -> c}| - +----------------------+ - - Example 4: map_from_entries with empty array + >>> df1 = spark.createDataFrame([1,2,2,3], "INT") + >>> df1 = df1.agg(sf.hll_sketch_agg("value").alias("sketch")) + >>> df2 = spark.createDataFrame([4,5,5,6], "INT") + >>> df2 = df2.agg(sf.hll_sketch_agg("value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch"))).show() + +-------------------------------------------------+ + |hll_sketch_estimate(hll_union_agg(sketch, false))| + +-------------------------------------------------+ + | 6| + +-------------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType( - ... StructType([ - ... StructField("key", IntegerType()), - ... StructField("value", StringType()) - ... ]) - ... ), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {}| - +----------------------+ + >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch", False))).show() + +-------------------------------------------------+ + |hll_sketch_estimate(hll_union_agg(sketch, false))| + +-------------------------------------------------+ + | 6| + +-------------------------------------------------+ """ - return _invoke_function_over_columns("map_from_entries", col) + if allowDifferentLgConfigK is None: + return _invoke_function_over_columns("hll_union_agg", col) + else: + return _invoke_function_over_columns("hll_union_agg", col, lit(allowDifferentLgConfigK)) @_try_remote_functions -def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Column: +def theta_sketch_agg( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, +) -> Column: """ - Array function: creates an array containing a column repeated count times. - - .. versionadded:: 2.4.0 + Aggregate function: returns the compact binary representation of the Datasketches + ThetaSketch with the values in the input column configured with lgNomEntries nominal entries. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the element to be repeated. - A column of any type. - count : :class:`~pyspark.sql.Column` or str or int - The name of the column, an expression, - or an integer that represents the number of times to repeat the element. + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an array, binary, double, float, integer, long, or string. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries, where nominal entries is the size of the sketch + (must be between 4 and 26, defaults to 12). A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an array of repeated elements. - Returns a column that evaluates to an array. + The binary representation of the ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.array` + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.theta_difference` + :meth:`pyspark.sql.functions.theta_union_agg` + :meth:`pyspark.sql.functions.theta_intersection_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- - Example 1: Usage with string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('ab',)], ['data']) - >>> df.select(sf.array_repeat(df.data, 3)).show() - +---------------------+ - |array_repeat(data, 3)| - +---------------------+ - | [ab, ab, ab]| - +---------------------+ + >>> df = spark.createDataFrame([1,2,2,3], "INT") + >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_sketch_agg(value, 12))| + +--------------------------------------------------+ + | 3| + +--------------------------------------------------+ - Example 2: Usage with integer + >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value", 15))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_sketch_agg(value, 15))| + +--------------------------------------------------+ + | 3| + +--------------------------------------------------+ + """ + fn = "theta_sketch_agg" + if lgNomEntries is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(3,)], ['data']) - >>> df.select(sf.array_repeat(df.data, 2)).show() - +---------------------+ - |array_repeat(data, 2)| - +---------------------+ - | [3, 3]| - +---------------------+ - Example 3: Usage with array +@_try_remote_functions +def theta_union_agg( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + ThetaSketch that is the union of the Theta sketches in the input column. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana'],)], ['data']) - >>> df.select(sf.array_repeat(df.data, 2)).show(truncate=False) - +----------------------------------+ - |array_repeat(data, 2) | - +----------------------------------+ - |[[apple, banana], [apple, banana]]| - +----------------------------------+ + .. versionadded:: 4.1.0 - Example 4: Usage with null + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries for the union operation + (must be between 4 and 26, defaults to 12) + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the merged ThetaSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", IntegerType(), True) - ... ]) - >>> df = spark.createDataFrame([(None, )], schema=schema) - >>> df.select(sf.array_repeat(df.data, 3)).show() - +---------------------+ - |array_repeat(data, 3)| - +---------------------+ - | [NULL, NULL, NULL]| - +---------------------+ + >>> df1 = spark.createDataFrame([1,2,2,3], "INT") + >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df2 = spark.createDataFrame([4,5,5,6], "INT") + >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.theta_sketch_estimate(sf.theta_union_agg("sketch"))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_union_agg(sketch, 12))| + +--------------------------------------------------+ + | 6| + +--------------------------------------------------+ """ - count = _enum_to_value(count) - count = lit(count) if isinstance(count, int) else count - - return _invoke_function_over_columns("array_repeat", col, count) + fn = "theta_union_agg" + if lgNomEntries is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) @_try_remote_functions -def arrays_zip(*cols: "ColumnOrName") -> Column: +def theta_intersection_agg(col: "ColumnOrName") -> Column: """ - Array function: Returns a merged array of structs in which the N-th struct contains all - N-th values of input arrays. If one of the arrays is shorter than others then - the resulting struct type value will be a `null` for missing elements. - - .. versionadded:: 2.4.0 + Aggregate function: returns the compact binary representation of the Datasketches + ThetaSketch that is the intersection of the Theta sketches in the input column - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - Columns of arrays to be merged. - A column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name Returns ------- :class:`~pyspark.sql.Column` - Merged array of entries. - Returns a column that evaluates to an array. + The binary representation of the intersected ThetaSketch. - Examples + See Also -------- - Example 1: Zipping two arrays of the same length + :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3], ['a', 'b', 'c'])], ['nums', 'letters']) - >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) - +-------------------------+ - |arrays_zip(nums, letters)| - +-------------------------+ - |[{1, a}, {2, b}, {3, c}] | - +-------------------------+ - + >>> df1 = spark.createDataFrame([1,2,2,3], "INT") + >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df2 = spark.createDataFrame([2,3,3,4], "INT") + >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.theta_sketch_estimate(sf.theta_intersection_agg("sketch"))).show() + +-----------------------------------------------------+ + |theta_sketch_estimate(theta_intersection_agg(sketch))| + +-----------------------------------------------------+ + | 2| + +-----------------------------------------------------+ + """ + fn = "theta_intersection_agg" + return _invoke_function_over_columns(fn, col) - Example 2: Zipping arrays of different lengths - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2], ['a', 'b', 'c'])], ['nums', 'letters']) - >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) - +---------------------------+ - |arrays_zip(nums, letters) | - +---------------------------+ - |[{1, a}, {2, b}, {NULL, c}]| - +---------------------------+ +@_try_remote_functions +def tuple_sketch_agg_double( + key: "ColumnOrName", + summary: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch with double summaries built from the key and summary columns. - Example 3: Zipping more than two arrays + .. versionadded:: 4.2.0 - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [([1, 2], ['a', 'b'], [True, False])], ['nums', 'letters', 'bools']) - >>> df.select(sf.arrays_zip(df.nums, df.letters, df.bools)).show(truncate=False) - +--------------------------------+ - |arrays_zip(nums, letters, bools)| - +--------------------------------+ - |[{1, a, true}, {2, b, false}] | - +--------------------------------+ + Parameters + ---------- + key : :class:`~pyspark.sql.Column` or column name + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. + summary : :class:`~pyspark.sql.Column` or column name + The column containing double summary values. + A column that evaluates to a double. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" - Example 4: Zipping arrays with null values + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` + :meth:`pyspark.sql.functions.tuple_sketch_summary_double` + :meth:`pyspark.sql.functions.tuple_union_agg_double` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, None], ['a', None, 'c'])], ['nums', 'letters']) - >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) - +------------------------------+ - |arrays_zip(nums, letters) | - +------------------------------+ - |[{1, a}, {2, NULL}, {NULL, c}]| - +------------------------------+ + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +--------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| + +--------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------+ """ - return _invoke_function_over_seq_of_columns("arrays_zip", cols) - - -@overload -def map_concat(*cols: "ColumnOrName") -> Column: ... - + fn = "tuple_sketch_agg_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) -@overload -def map_concat(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... + return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) @_try_remote_functions -def map_concat( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], +def tuple_sketch_agg_integer( + key: "ColumnOrName", + summary: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, ) -> Column: """ - Map function: Returns the union of all given maps. - - .. versionadded:: 2.4.0 + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch with integer summaries built from the key and summary columns. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - Column names or :class:`~pyspark.sql.Column` + key : :class:`~pyspark.sql.Column` or column name + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. + summary : :class:`~pyspark.sql.Column` or column name + The column containing integer summary values. + A column that evaluates to an integer. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - A map of merged entries from other maps. + The binary representation of the TupleSketch. - Notes - ----- - For duplicate keys in input maps, the handling is governed by `spark.sql.mapKeyDedupPolicy`. - By default, it throws an exception. If set to `LAST_WIN`, it uses the last map's value. + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` + :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` + :meth:`pyspark.sql.functions.tuple_union_agg_integer` Examples -------- - Example 1: Basic usage of map_concat - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +------------------------+ - |map_concat(map1, map2) | - +------------------------+ - |{1 -> a, 2 -> b, 3 -> c}| - +------------------------+ + >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +----------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| + +----------------------------------------------------------------------------+ + | 2.0| + +----------------------------------------------------------------------------+ + """ + fn = "tuple_sketch_agg_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) - Example 2: map_concat with overlapping keys + return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) - >>> from pyspark.sql import functions as sf - >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(2, 'c', 3, 'd') as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +------------------------+ - |map_concat(map1, map2) | - +------------------------+ - |{1 -> a, 2 -> c, 3 -> d}| - +------------------------+ - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) - Example 3: map_concat with three maps +@_try_remote_functions +def tuple_union_agg_double( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the union of the double TupleSketch objects in the input column. - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a') as map1, map(2, 'b') as map2, map(3, 'c') as map3") - >>> df.select(sf.map_concat("map1", "map2", "map3")).show(truncate=False) - +----------------------------+ - |map_concat(map1, map2, map3)| - +----------------------------+ - |{1 -> a, 2 -> b, 3 -> c} | - +----------------------------+ + .. versionadded:: 4.2.0 - Example 4: map_concat with empty map + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map() as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +----------------------+ - |map_concat(map1, map2)| - +----------------------+ - |{1 -> a, 2 -> b} | - +----------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the merged TupleSketch. - Example 5: map_concat with null values + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_double` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, null) as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +---------------------------+ - |map_concat(map1, map2) | - +---------------------------+ - |{1 -> a, 2 -> b, 3 -> NULL}| - +---------------------------+ + >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(3, 30.0), (4, 40.0)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_union_agg_double("sketch"))).show() + +---------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_union_agg_double(sketch, 12, sum))| + +---------------------------------------------------------------------+ + | 4.0| + +---------------------------------------------------------------------+ """ - if len(cols) == 1 and isinstance(cols[0], (list, set)): - cols = cols[0] # type: ignore[assignment] - return _invoke_function_over_seq_of_columns("map_concat", cols) # type: ignore[arg-type] + fn = "tuple_union_agg_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) @_try_remote_functions -def sequence( - start: "ColumnOrName", stop: "ColumnOrName", step: Optional["ColumnOrName"] = None +def tuple_union_agg_integer( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, ) -> Column: """ - Array function: Generate a sequence of integers from `start` to `stop`, incrementing by `step`. - If `step` is not set, the function increments by 1 if `start` is less than or equal to `stop`, - otherwise it decrements by 1. - - .. versionadded:: 2.4.0 + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the union of the integer TupleSketch objects in the input column. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- - start : :class:`~pyspark.sql.Column` or str - The starting value (inclusive) of the sequence. - A column that evaluates to an integral, date, or timestamp. - stop : :class:`~pyspark.sql.Column` or str - The last value (inclusive) of the sequence. - A column that evaluates to an integral, date, or timestamp. - step : :class:`~pyspark.sql.Column` or str, optional - The value to add to the current element to get the next element in the sequence. - The default is 1 if `start` is less than or equal to `stop`, otherwise -1. - A column that evaluates to an integral or interval. + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an array of sequence values. - Returns a column that evaluates to an array. + The binary representation of the merged TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_integer` Examples -------- - Example 1: Generating a sequence with default step + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(3, 30), (4, 40)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_union_agg_integer("sketch"))).show() + +-----------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch, 12, sum))| + +-----------------------------------------------------------------------+ + | 4.0| + +-----------------------------------------------------------------------+ + """ + fn = "tuple_union_agg_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(-2, 2)], ['start', 'stop']) - >>> df.select(sf.sequence(df.start, df.stop)).show() - +---------------------+ - |sequence(start, stop)| - +---------------------+ - | [-2, -1, 0, 1, 2]| - +---------------------+ + return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) - Example 2: Generating a sequence with a custom step - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(4, -4, -2)], ['start', 'stop', 'step']) - >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() - +---------------------------+ - |sequence(start, stop, step)| - +---------------------------+ - | [4, 2, 0, -2, -4]| - +---------------------------+ +@_try_remote_functions +def tuple_intersection_agg_double( + col: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the intersection of the double TupleSketch objects in the input column. + .. versionadded:: 4.2.0 - Example 3: Generating a sequence with a negative step + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(5, 1, -1)], ['start', 'stop', 'step']) - >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() - +---------------------------+ - |sequence(start, stop, step)| - +---------------------------+ - | [5, 4, 3, 2, 1]| - +---------------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the intersected TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_intersection_double` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0), (3, 30.0)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(2, 40.0), (3, 50.0), (4, 60.0)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_intersection_agg_double("sketch"))).show() + +------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch, sum))| + +------------------------------------------------------------------------+ + | 2.0| + +------------------------------------------------------------------------+ """ - if step is None: - return _invoke_function_over_columns("sequence", start, stop) + fn = "tuple_intersection_agg_double" + if mode is None: + return _invoke_function_over_columns(fn, col) else: - return _invoke_function_over_columns("sequence", start, stop, step) + return _invoke_function_over_columns(fn, col, lit(mode)) @_try_remote_functions -def from_csv( +def tuple_intersection_agg_integer( col: "ColumnOrName", - schema: Union[Column, str], - options: Optional[Mapping[str, str]] = None, + mode: Optional[Union[str, Column]] = None, ) -> Column: """ - CSV Function: Parses a column containing a CSV string into a row with the specified schema. - Returns `null` if the string cannot be parsed. - - .. versionadded:: 3.0.0 + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the intersection of the integer TupleSketch objects in the input column. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - A column or column name in CSV format. - schema : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string, or a DDL-formatted type string, or a DataType. - A column, or Python string literal with schema in DDL format, to use when parsing the CSV column. - options : dict, optional - Options to control parsing. Accepts the same options as the CSV datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - A column of parsed CSV values. - Returns a column that evaluates to a struct. + The binary representation of the intersected TupleSketch. - Examples + See Also -------- - Example 1: Parsing a simple CSV string + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_intersection_integer` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> data = [("1,2,3",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, 3}| - +---------------+ + >>> df1 = spark.createDataFrame([(1, 10), (2, 20), (3, 30)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(2, 40), (3, 50), (4, 60)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_agg_integer("sketch"))).show() + +--------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch, sum))| + +--------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------+ + """ + fn = "tuple_intersection_agg_integer" + if mode is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(mode)) - Example 2: Using schema_of_csv to infer the schema - >>> from pyspark.sql import functions as sf - >>> data = [("1,2,3",)] - >>> value = data[0][0] - >>> df.select(sf.from_csv(df.value, sf.schema_of_csv(value))).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, 3}| - +---------------+ +@_try_remote_functions +def kll_sketch_agg_bigint( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + KllLongsSketch built with the values in the input column. The optional k parameter + controls the size and accuracy of the sketch (default 200, range 8-65535). - Example 3: Ignoring leading white space in the CSV string + .. versionadded:: 4.1.0 - >>> from pyspark.sql import functions as sf - >>> data = [(" abc",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> options = {'ignoreLeadingWhiteSpace': True} - >>> df.select(sf.from_csv(df.value, "s string", options)).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {abc}| - +---------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The column containing bigint values to aggregate. + A column that evaluates to an integral. + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (default 200, range 8-65535) + A column that evaluates to an integer. Must be a constant. - Example 4: Parsing a CSV string with a missing value + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the KllLongsSketch. + Examples + -------- >>> from pyspark.sql import functions as sf - >>> data = [("1,2,",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, NULL}| - +---------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> result = df.agg(sf.kll_sketch_agg_bigint("value")).first()[0] + >>> result is not None and len(result) > 0 + True + """ + fn = "kll_sketch_agg_bigint" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) - Example 5: Parsing a CSV string with a different delimiter - >>> from pyspark.sql import functions as sf - >>> data = [("1;2;3",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> options = {'delimiter': ';'} - >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT", options)).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, 3}| - +---------------+ +@_try_remote_functions +def kll_sketch_agg_float( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, +) -> Column: """ - from pyspark.sql.classic.column import _to_java_column + Aggregate function: returns the compact binary representation of the Datasketches + KllFloatsSketch built with the values in the input column. The optional k parameter + controls the size and accuracy of the sketch (default 200, range 8-65535). - if not isinstance(schema, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "schema", - "arg_type": type(schema).__name__, - }, - ) - - return _invoke_function( - "from_csv", _to_java_column(col), _to_java_column(lit(schema)), _options_to_str(options) - ) - - -def _unresolved_named_lambda_variable(name: str) -> Column: - """ - Create `o.a.s.sql.expressions.UnresolvedNamedLambdaVariable`, - convert it to o.s.sql.Column and wrap in Python `Column` - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- - name_parts : str - """ - from py4j.java_gateway import JVMView - - sc = _get_active_spark_context() - return Column(cast(JVMView, sc._jvm).PythonSQLUtils.unresolvedNamedLambdaVariable(name)) - - -def _get_lambda_parameters(f: Callable) -> ValuesView[inspect.Parameter]: - signature = inspect.signature(f) - parameters = signature.parameters.values() - - # We should exclude functions that use - # variable args and keyword argnames - # as well as keyword only args - supported_parameter_types = { - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.POSITIONAL_ONLY, - } - - # Validate that - # function arity is between 1 and 3 - if not (1 <= len(parameters) <= 3): - raise PySparkValueError( - errorClass="WRONG_NUM_ARGS_FOR_HIGHER_ORDER_FUNCTION", - messageParameters={"func_name": f.__name__, "num_args": str(len(parameters))}, - ) - - # and all arguments can be used as positional - if not all(p.kind in supported_parameter_types for p in parameters): - raise PySparkValueError( - errorClass="UNSUPPORTED_PARAM_TYPE_FOR_HIGHER_ORDER_FUNCTION", - messageParameters={"func_name": f.__name__}, - ) - - return parameters - - -def _create_lambda(f: Callable) -> Callable: - """ - Create `o.a.s.sql.expressions.LambdaFunction` corresponding - to transformation described by f - - :param f: A Python of one of the following forms: - - (Column) -> Column: ... - - (Column, Column) -> Column: ... - - (Column, Column, Column) -> Column: ... - """ - from py4j.java_gateway import JVMView - - from pyspark.sql.classic.column import _to_seq - - parameters = _get_lambda_parameters(f) - - sc = _get_active_spark_context() - - argnames = ["x", "y", "z"] - args = [_unresolved_named_lambda_variable(arg) for arg in argnames[: len(parameters)]] - - result = f(*args) - - if not isinstance(result, Column): - raise PySparkValueError( - errorClass="HIGHER_ORDER_FUNCTION_SHOULD_RETURN_COLUMN", - messageParameters={"func_name": f.__name__, "return_type": type(result).__name__}, - ) - - jexpr = result._jc - jargs = _to_seq(sc, [arg._jc for arg in args]) - return cast(JVMView, sc._jvm).PythonSQLUtils.lambdaFunction(jexpr, jargs) - - -def _invoke_higher_order_function( - name: str, - cols: Sequence["ColumnOrName"], - funs: Sequence[Callable], -) -> Column: - """ - Invokes expression identified by name, - (relative to ```org.apache.spark.sql.catalyst.expressions``) - and wraps the result with Column (first Scala one, then Python). + col : :class:`~pyspark.sql.Column` or column name + The column containing float values to aggregate + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (default 200, range 8-65535) + A column that evaluates to an integer. Must be a constant. - :param name: Name of the expression - :param cols: a list of columns - :param funs: a list of (*Column) -> Column functions. + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the KllFloatsSketch. - :return: a Column + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> result = df.agg(sf.kll_sketch_agg_float("value")).first()[0] + >>> result is not None and len(result) > 0 + True """ - from py4j.java_gateway import JVMView - - from pyspark.sql.classic.column import _to_java_column, _to_seq - - sc = _get_active_spark_context() - jfuns = [_create_lambda(f) for f in funs] - jcols = [_to_java_column(c) for c in cols] - return Column(cast(JVMView, sc._jvm).PythonSQLUtils.fn(name, _to_seq(sc, jcols + jfuns))) - - -@overload -def transform(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... - - -@overload -def transform(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... + fn = "kll_sketch_agg_float" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def transform( +def kll_sketch_agg_double( col: "ColumnOrName", - f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], + k: Optional[Union[int, Column]] = None, ) -> Column: """ - Returns an array of elements after applying a transformation to each element in the input array. - - .. versionadded:: 3.1.0 + Aggregate function: returns the compact binary representation of the Datasketches + KllDoublesSketch built with the values in the input column. The optional k parameter + controls the size and accuracy of the sketch (default 200, range 8-65535). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - a function that is applied to each element of the input array. - Can take one of the following forms: - - - Unary ``(x: Column) -> Column: ...`` - - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is - a 0-based index of the element. - - and can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + col : :class:`~pyspark.sql.Column` or column name + The column containing double values to aggregate. + A column that evaluates to a float or double. + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (default 200, range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - a new array of transformed elements. - Returns a column that evaluates to an array. + The binary representation of the KllDoublesSketch. Examples -------- - >>> df = spark.createDataFrame([(1, [1, 2, 3, 4])], ("key", "values")) - >>> df.select(transform("values", lambda x: x * 2).alias("doubled")).show() - +------------+ - | doubled| - +------------+ - |[2, 4, 6, 8]| - +------------+ - - >>> def alternate(x, i): - ... return when(i % 2 == 0, x).otherwise(-x) - ... - >>> df.select(transform("values", alternate).alias("alternated")).show() - +--------------+ - | alternated| - +--------------+ - |[1, -2, 3, -4]| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> result = df.agg(sf.kll_sketch_agg_double("value")).first()[0] + >>> result is not None and len(result) > 0 + True """ - return _invoke_higher_order_function("transform", [col], [f]) + fn = "kll_sketch_agg_double" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: +def kll_merge_agg_bigint( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, +) -> Column: """ - Returns whether a predicate holds for one or more elements in the array. - - .. versionadded:: 3.1.0 + Aggregate function: merges binary KllLongsSketch representations and returns the + merged sketch. The optional k parameter controls the size and accuracy of the merged + sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value + from the first input sketch. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.2 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - ``(x: Column) -> Column: ...`` returning the Boolean expression. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + col : :class:`~pyspark.sql.Column` or column name + The column containing binary KllLongsSketch representations + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - True if "any" element of an array evaluates to True when passed as an argument to - given function and False otherwise. - Returns a column that evaluates to a boolean. + The merged binary representation of the KllLongsSketch. Examples -------- - >>> df = spark.createDataFrame([(1, [1, 2, 3, 4]), (2, [3, -1, 0])],("key", "values")) - >>> df.select(exists("values", lambda x: x < 0).alias("any_negative")).show() - +------------+ - |any_negative| - +------------+ - | false| - | true| - +------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1,2,3], "INT") + >>> df2 = spark.createDataFrame([4,5,6], "INT") + >>> sketch1 = df1.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch2 = df2.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_bigint("sketch").alias("merged")) + >>> n = merged.select(sf.kll_sketch_get_n_bigint("merged")).first()[0] + >>> n + 6 """ - return _invoke_higher_order_function("exists", [col], [f]) + fn = "kll_merge_agg_bigint" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: +def kll_merge_agg_float( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, +) -> Column: """ - Returns whether a predicate holds for every element in the array. - - .. versionadded:: 3.1.0 + Aggregate function: merges binary KllFloatsSketch representations and returns the + merged sketch. The optional k parameter controls the size and accuracy of the merged + sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value + from the first input sketch. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.2 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - ``(x: Column) -> Column: ...`` returning the Boolean expression. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + col : :class:`~pyspark.sql.Column` or column name + The column containing binary KllFloatsSketch representations + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - True if "all" elements of an array evaluates to True when passed as an argument to - given function and False otherwise. - Returns a column that evaluates to a boolean. + The merged binary representation of the KllFloatsSketch. Examples -------- - >>> df = spark.createDataFrame( - ... [(1, ["bar"]), (2, ["foo", "bar"]), (3, ["foobar", "foo"])], - ... ("key", "values") - ... ) - >>> df.select(forall("values", lambda x: x.rlike("foo")).alias("all_foo")).show() - +-------+ - |all_foo| - +-------+ - | false| - | false| - | true| - +-------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "FLOAT") + >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "FLOAT") + >>> sketch1 = df1.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch2 = df2.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_float("sketch").alias("merged")) + >>> n = merged.select(sf.kll_sketch_get_n_float("merged")).first()[0] + >>> n + 6 """ - return _invoke_higher_order_function("forall", [col], [f]) - - -@overload -def filter(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... - - -@overload -def filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... + fn = "kll_merge_agg_float" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def filter( +def kll_merge_agg_double( col: "ColumnOrName", - f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], + k: Optional[Union[int, Column]] = None, ) -> Column: """ - Returns an array of elements for which a predicate holds in a given array. - - .. versionadded:: 3.1.0 + Aggregate function: merges binary KllDoublesSketch representations and returns the + merged sketch. The optional k parameter controls the size and accuracy of the merged + sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value + from the first input sketch. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.2 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - A function that returns the Boolean expression. - Can take one of the following forms: - - - Unary ``(x: Column) -> Column: ...`` - - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is - a 0-based index of the element. - - and can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + col : :class:`~pyspark.sql.Column` or column name + The column containing binary KllDoublesSketch representations + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - filtered array of elements where given function evaluated to True - when passed as an argument. - Returns a column that evaluates to an array. + The merged binary representation of the KllDoublesSketch. Examples -------- - >>> df = spark.createDataFrame( - ... [(1, ["2018-09-20", "2019-02-03", "2019-07-01", "2020-06-01"])], - ... ("key", "values") - ... ) - >>> def after_second_quarter(x): - ... return month(to_date(x)) > 6 - ... - >>> df.select( - ... filter("values", after_second_quarter).alias("after_second_quarter") - ... ).show(truncate=False) - +------------------------+ - |after_second_quarter | - +------------------------+ - |[2018-09-20, 2019-07-01]| - +------------------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "DOUBLE") + >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "DOUBLE") + >>> sketch1 = df1.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch2 = df2.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_double("sketch").alias("merged")) + >>> n = merged.select(sf.kll_sketch_get_n_double("merged")).first()[0] + >>> n + 6 """ - return _invoke_higher_order_function("filter", [col], [f]) + fn = "kll_merge_agg_double" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def aggregate( - col: "ColumnOrName", - initialValue: "ColumnOrName", - merge: Callable[[Column, Column], Column], - finish: Optional[Callable[[Column], Column]] = None, -) -> Column: +def bitmap_construct_agg(col: "ColumnOrName") -> Column: """ - Applies a binary operator to an initial state and all elements in the array, - and reduces this to a single state. The final state is converted into the final result - by applying a finish function. - - Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). - - .. versionadded:: 3.1.0 + Returns a bitmap with the positions of the bits set from all the values from the input column. + The input column will most likely be bitmap_bit_position(). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression. - A column of any type. - merge : function - a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``initialValue``. - finish : function, optional - an optional unary function ``(x: Column) -> Column: ...`` - used to convert accumulated value. + col : :class:`~pyspark.sql.Column` or column name + The input column will most likely be bitmap_bit_position(). + A column that evaluates to a long. - Returns - ------- - :class:`~pyspark.sql.Column` - final value after aggregate function is applied. - Returns a column of the same type as ``initialValue``. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` + :meth:`pyspark.sql.functions.bitmap_and_agg` Examples -------- - >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) - >>> df.select(aggregate("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() - +----+ - | sum| - +----+ - |42.0| - +----+ - - >>> def merge(acc, x): - ... count = acc.count + 1 - ... sum = acc.sum + x - ... return struct(count.alias("count"), sum.alias("sum")) - ... + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,),(2,),(3,)], ["a"]) >>> df.select( - ... aggregate( - ... "values", - ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), - ... merge, - ... lambda acc: acc.sum / acc.count, - ... ).alias("mean") + ... sf.bitmap_construct_agg(sf.bitmap_bit_position('a')) ... ).show() - +----+ - |mean| - +----+ - | 8.4| - +----+ + +--------------------------------------------+ + |bitmap_construct_agg(bitmap_bit_position(a))| + +--------------------------------------------+ + | [07 00 00 00 00 0...| + +--------------------------------------------+ """ - if finish is not None: - return _invoke_higher_order_function("aggregate", [col, initialValue], [merge, finish]) - - else: - return _invoke_higher_order_function("aggregate", [col, initialValue], [merge]) + return _invoke_function_over_columns("bitmap_construct_agg", col) @_try_remote_functions -def reduce( - col: "ColumnOrName", - initialValue: "ColumnOrName", - merge: Callable[[Column, Column], Column], - finish: Optional[Callable[[Column], Column]] = None, -) -> Column: +def bitmap_or_agg(col: "ColumnOrName") -> Column: """ - Applies a binary operator to an initial state and all elements in the array, - and reduces this to a single state. The final state is converted into the final result - by applying a finish function. - - Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. + The input column should be bitmaps created from bitmap_construct_agg(). .. versionadded:: 3.5.0 + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_and_agg` + Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression. - A column of any type. - merge : function - a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``zero``. - finish : function, optional - an optional unary function ``(x: Column) -> Column: ...`` - used to convert accumulated value. - - Returns - ------- - :class:`~pyspark.sql.Column` - final value after aggregate function is applied. - Returns a column of the same type as ``initialValue``. + col : :class:`~pyspark.sql.Column` or column name + The input column should be bitmaps created from bitmap_construct_agg(). Examples -------- - >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) - >>> df.select(reduce("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() - +----+ - | sum| - +----+ - |42.0| - +----+ - - >>> def merge(acc, x): - ... count = acc.count + 1 - ... sum = acc.sum + x - ... return struct(count.alias("count"), sum.alias("sum")) - ... - >>> df.select( - ... reduce( - ... "values", - ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), - ... merge, - ... lambda acc: acc.sum / acc.count, - ... ).alias("mean") - ... ).show() - +----+ - |mean| - +----+ - | 8.4| - +----+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("10",),("20",),("40",)], ["a"]) + >>> df.select(sf.bitmap_or_agg(sf.to_binary(df.a, sf.lit("hex")))).show() + +--------------------------------+ + |bitmap_or_agg(to_binary(a, hex))| + +--------------------------------+ + | [70 00 00 00 00 0...| + +--------------------------------+ """ - if finish is not None: - return _invoke_higher_order_function("reduce", [col, initialValue], [merge, finish]) - - else: - return _invoke_higher_order_function("reduce", [col, initialValue], [merge]) + return _invoke_function_over_columns("bitmap_or_agg", col) @_try_remote_functions -def zip_with( - left: "ColumnOrName", - right: "ColumnOrName", - f: Callable[[Column, Column], Column], -) -> Column: +def bitmap_and_agg(col: "ColumnOrName") -> Column: """ - Merge two given arrays, element-wise, into a single array using a function. - If one array is shorter, nulls are appended at the end to match the length of the longer - array, before applying the function. + Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. + The input column should be bitmaps created from bitmap_construct_agg(). - .. versionadded:: 3.1.0 + .. versionadded:: 4.1.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` Parameters ---------- - left : :class:`~pyspark.sql.Column` or str - name of the first column or expression. - A column that evaluates to an array. - right : :class:`~pyspark.sql.Column` or str - name of the second column or expression. - A column that evaluates to an array. - f : function - a binary function ``(x1: Column, x2: Column) -> Column...`` - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). - - Returns - ------- - :class:`~pyspark.sql.Column` - array of calculated values derived by applying given function to each pair of arguments. - Returns a column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + The input column should be bitmaps created from bitmap_construct_agg(). Examples -------- - >>> df = spark.createDataFrame([(1, [1, 3, 5, 8], [0, 2, 4, 6])], ("id", "xs", "ys")) - >>> df.select(zip_with("xs", "ys", lambda x, y: x ** y).alias("powers")).show(truncate=False) - +---------------------------+ - |powers | - +---------------------------+ - |[1.0, 9.0, 625.0, 262144.0]| - +---------------------------+ - - >>> df = spark.createDataFrame([(1, ["foo", "bar"], [1, 2, 3])], ("id", "xs", "ys")) - >>> df.select(zip_with("xs", "ys", lambda x, y: concat_ws("_", x, y)).alias("xs_ys")).show() - +-----------------+ - | xs_ys| - +-----------------+ - |[foo_1, bar_2, 3]| - +-----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("F0",),("70",),("30",)], ["a"]) + >>> df.select(sf.bitmap_and_agg(sf.to_binary(df.a, sf.lit("hex")))).show() + +---------------------------------+ + |bitmap_and_agg(to_binary(a, hex))| + +---------------------------------+ + | [30 00 00 00 00 0...| + +---------------------------------+ """ - return _invoke_higher_order_function("zip_with", [left, right], [f]) + return _invoke_function_over_columns("bitmap_and_agg", col) @_try_remote_functions -def transform_keys(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: +def bitmap_xor_agg(col: "ColumnOrName") -> Column: """ - Applies a function to every key-value pair in a map and returns - a map with the results of those applications as the new keys for the pairs. + Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. + The input column should be bitmaps created from bitmap_construct_agg(). - .. versionadded:: 3.1.0 + .. versionadded:: 4.4.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` + :meth:`pyspark.sql.functions.bitmap_and_agg` Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression - f : function - a binary function ``(k: Column, v: Column) -> Column...`` - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). - - Returns - ------- - :class:`~pyspark.sql.Column` - a new map of entries where new keys were calculated by applying given function to - each key value argument. + col : :class:`~pyspark.sql.Column` or column name + The input column should be bitmaps created from bitmap_construct_agg(). Examples -------- - >>> df = spark.createDataFrame([(1, {"foo": -2.0, "bar": 2.0})], ("id", "data")) - >>> row = df.select(transform_keys( - ... "data", lambda k, _: upper(k)).alias("data_upper") - ... ).head() - >>> sorted(row["data_upper"].items()) - [('BAR', 2.0), ('FOO', -2.0)] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("10",), ("30",), ("40",)], ["a"]) + >>> df.select(sf.bitmap_xor_agg(sf.to_binary(df.a, sf.lit("hex")))).show() + +---------------------------------+ + |bitmap_xor_agg(to_binary(a, hex))| + +---------------------------------+ + | [60 00 00 00 00 0...| + +---------------------------------+ """ - return _invoke_higher_order_function("transform_keys", [col], [f]) + return _invoke_function_over_columns("bitmap_xor_agg", col) + + +# ---------------------- Window Functions ---------------------- @_try_remote_functions -def transform_values(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: +def row_number() -> Column: """ - Applies a function to every key-value pair in a map and returns - a map with the results of those applications as the new values for the pairs. + Window function: returns a sequential number starting at 1 within a window partition. - .. versionadded:: 3.1.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression - f : function - a binary function ``(k: Column, v: Column) -> Column...`` - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). - Returns ------- :class:`~pyspark.sql.Column` - a new map of entries where new values were calculated by applying given function to - each key value argument. + the column for calculating row numbers. + + See Also + -------- + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` Examples -------- - >>> df = spark.createDataFrame([(1, {"IT": 10.0, "SALES": 2.0, "OPS": 24.0})], ("id", "data")) - >>> row = df.select(transform_values( - ... "data", lambda k, v: when(k.isin("IT", "OPS"), v + 10.0).otherwise(v) - ... ).alias("new_data")).head() - >>> sorted(row["new_data"].items()) - [('IT', 20.0), ('OPS', 34.0), ('SALES', 2.0)] + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.range(3) + >>> w = Window.orderBy(df.id.desc()) + >>> df.withColumn("desc_order", sf.row_number().over(w)).show() + +---+----------+ + | id|desc_order| + +---+----------+ + | 2| 1| + | 1| 2| + | 0| 3| + +---+----------+ """ - return _invoke_higher_order_function("transform_values", [col], [f]) + return _invoke_function("row_number") @_try_remote_functions -def map_filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: +def dense_rank() -> Column: """ - Collection function: Returns a new map column whose key-value pairs satisfy a given - predicate function. + Window function: returns the rank of rows within a window partition, without any gaps. - .. versionadded:: 3.1.0 + The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking + sequence when there are ties. That is, if you were ranking a competition using dense_rank + and had three people tie for second place, you would say that all three were in second + place and that the next person came in third. Rank would give me sequential numbers, making + the person that came in third place (after the ties) would register as coming in fifth. + + This is equivalent to the DENSE_RANK function in SQL. + + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or a column expression representing the map to be filtered. - f : function - A binary function ``(k: Column, v: Column) -> Column...`` that defines the predicate. - This function should return a boolean column that will be used to filter the input map. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). - Returns ------- :class:`~pyspark.sql.Column` - A new map column containing only the key-value pairs that satisfy the predicate. + the column for calculating ranks. - Examples + See Also -------- - Example 1: Filtering a map with a simple condition + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) - >>> row = df.select( - ... sf.map_filter("data", lambda _, v: v > 30.0).alias("data_filtered") - ... ).head() - >>> sorted(row["data_filtered"].items()) - [('baz', 32.0), ('foo', 42.0)] + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("drank", sf.dense_rank().over(w)).show() + +-----+-----+ + |value|drank| + +-----+-----+ + | 1| 1| + | 1| 1| + | 2| 2| + | 3| 3| + | 3| 3| + | 4| 4| + +-----+-----+ + """ + return _invoke_function("dense_rank") - Example 2: Filtering a map with a condition on keys - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) - >>> row = df.select( - ... sf.map_filter("data", lambda k, _: k.startswith("b")).alias("data_filtered") - ... ).head() - >>> sorted(row["data_filtered"].items()) - [('bar', 1.0), ('baz', 32.0)] +@_try_remote_functions +def rank() -> Column: + """ + Window function: returns the rank of rows within a window partition. - Example 3: Filtering a map with a complex condition + The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking + sequence when there are ties. That is, if you were ranking a competition using dense_rank + and had three people tie for second place, you would say that all three were in second + place and that the next person came in third. Rank would give me sequential numbers, making + the person that came in third place (after the ties) would register as coming in fifth. + + This is equivalent to the RANK function in SQL. + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for calculating ranks. + + See Also + -------- + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.row_number` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) - >>> row = df.select( - ... sf.map_filter("data", lambda k, v: k.startswith("b") & (v > 1.0)).alias("data_filtered") - ... ).head() - >>> sorted(row["data_filtered"].items()) - [('baz', 32.0)] + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("drank", sf.rank().over(w)).show() + +-----+-----+ + |value|drank| + +-----+-----+ + | 1| 1| + | 1| 1| + | 2| 3| + | 3| 4| + | 3| 4| + | 4| 6| + +-----+-----+ """ - return _invoke_higher_order_function("map_filter", [col], [f]) + return _invoke_function("rank") @_try_remote_functions -def map_zip_with( - col1: "ColumnOrName", - col2: "ColumnOrName", - f: Callable[[Column, Column, Column], Column], -) -> Column: +def counter_diff(value: "ColumnOrName", startTime: Optional["ColumnOrName"] = None) -> Column: """ - Collection: Merges two given maps into a single map by applying a function to - the key-value pairs. + Window function: computes the differences between consecutive cumulative counter values in a + time series, thereby converting the counter from the cumulative to the delta format. - .. versionadded:: 3.1.0 + Gracefully handles counter resets by returning NULL. Counter resets are detected when the + counter value decreases, or when the start time advances between rows. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Use the PARTITION BY clause of the window to separate independent counters. This is done by + specifying all columns which uniquely identify a time series. These are typically the counter + name and any attributes tied to the counter. + + Use the ORDER BY clause of the window to order the observations by the associated timestamp + in ascending order. + + .. versionadded:: 4.3.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - The name of the first column or a column expression representing the first map. - col2 : :class:`~pyspark.sql.Column` or str - The name of the second column or a column expression representing the second map. - f : function - A ternary function ``(k: Column, v1: Column, v2: Column) -> Column...`` that defines - how to merge the values from the two maps. This function should return a column that - will be used as the value in the resulting map. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + value : :class:`~pyspark.sql.Column` or column name + A cumulative counter. Must be a numeric data type. Must be non-negative. + startTime : :class:`~pyspark.sql.Column` or column name, optional + An optional timestamp parameter which indicates when the counter was last set to zero. + Used to signal counter resets. Returns ------- :class:`~pyspark.sql.Column` - A new map column where each key-value pair is the result of applying the function to - the corresponding key-value pairs in the input maps. + The difference between the current and previous counter value within the window partition. Examples -------- - Example 1: Merging two maps with a simple function - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], - ... ("id", "map1", "map2")) - >>> row = df.select( - ... sf.map_zip_with("map1", "map2", lambda _, v1, v2: v1 + v2).alias("updated_data") - ... ).head() - >>> sorted(row["updated_data"].items()) - [('A', 4), ('B', 6)] - - Example 2: Merging two maps with a complex function - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], - ... ("id", "map1", "map2")) - >>> row = df.select( - ... sf.map_zip_with("map1", "map2", - ... lambda k, v1, v2: sf.when(k == "A", v1 + v2).otherwise(v1 - v2) - ... ).alias("updated_data") - ... ).head() - >>> sorted(row["updated_data"].items()) - [('A', 4), ('B', -2)] - - Example 3: Merging two maps with mismatched keys + >>> from pyspark.sql import Window + >>> from datetime import datetime + >>> df = spark.createDataFrame( + ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100), + ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200), + ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400), + ... ('http_requests', datetime(2026, 1, 1, 0, 3), 50), + ... ('http_requests', datetime(2026, 1, 1, 0, 4), 100)], + ... "m STRING, t TIMESTAMP_NTZ, c INT") + >>> w = Window.partitionBy("m").orderBy("t") + >>> df.select("m", "t", "c", sf.counter_diff("c").over(w).alias("diff")).show() + +-------------+-------------------+---+----+ + | m| t| c|diff| + +-------------+-------------------+---+----+ + |http_requests|2026-01-01 00:00:00|100|NULL| + |http_requests|2026-01-01 00:01:00|200| 100| + |http_requests|2026-01-01 00:02:00|400| 200| + |http_requests|2026-01-01 00:03:00| 50|NULL| + |http_requests|2026-01-01 00:04:00|100| 50| + +-------------+-------------------+---+----+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1, {"A": 1, "B": 2}, {"B": 3, "C": 4})], - ... ("id", "map1", "map2")) - >>> row = df.select( - ... sf.map_zip_with("map1", "map2", - ... lambda _, v1, v2: sf.when(v2.isNull(), v1).otherwise(v1 + v2) - ... ).alias("updated_data") - ... ).head() - >>> sorted(row["updated_data"].items()) - [('A', 1), ('B', 5), ('C', None)] + >>> df2 = spark.createDataFrame( + ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100, datetime(2026, 1, 1, 0, 0)), + ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200, datetime(2026, 1, 1, 0, 0)), + ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400, datetime(2026, 1, 1, 0, 0)), + ... ('http_requests', datetime(2026, 1, 1, 0, 3), 500, datetime(2026, 1, 1, 0, 2, 15)), + ... ('http_requests', datetime(2026, 1, 1, 0, 4), 600, datetime(2026, 1, 1, 0, 2, 15))], + ... "m STRING, t TIMESTAMP_NTZ, c INT, s TIMESTAMP_NTZ") + >>> df2.select("m", "t", "s", "c", sf.counter_diff("c", "s").over(w).alias("diff")).show() + +-------------+-------------------+-------------------+---+----+ + | m| t| s| c|diff| + +-------------+-------------------+-------------------+---+----+ + |http_requests|2026-01-01 00:00:00|2026-01-01 00:00:00|100|NULL| + |http_requests|2026-01-01 00:01:00|2026-01-01 00:00:00|200| 100| + |http_requests|2026-01-01 00:02:00|2026-01-01 00:00:00|400| 200| + |http_requests|2026-01-01 00:03:00|2026-01-01 00:02:15|500|NULL| + |http_requests|2026-01-01 00:04:00|2026-01-01 00:02:15|600| 100| + +-------------+-------------------+-------------------+---+----+ """ - return _invoke_higher_order_function("map_zip_with", [col1, col2], [f]) + if startTime is None: + return _invoke_function_over_columns("counter_diff", value) + return _invoke_function_over_columns("counter_diff", value, startTime) @_try_remote_functions -def str_to_map( - text: "ColumnOrName", - pairDelim: Optional["ColumnOrName"] = None, - keyValueDelim: Optional["ColumnOrName"] = None, -) -> Column: +def cume_dist() -> Column: """ - Map function: Converts a string into a map after splitting the text into key/value pairs - using delimiters. Both `pairDelim` and `keyValueDelim` are treated as regular expressions. + Window function: returns the cumulative distribution of values within a window partition, + i.e. the fraction of rows that are below the current row. - .. versionadded:: 3.5.0 + .. versionadded:: 1.6.0 - Parameters - ---------- - text : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - pairDelim : :class:`~pyspark.sql.Column` or str, optional - Delimiter to use to split pairs. Default is comma (,). - A column that evaluates to a string. - keyValueDelim : :class:`~pyspark.sql.Column` or str, optional - Delimiter to use to split key/value. Default is colon (:). - A column that evaluates to a string. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Returns ------- :class:`~pyspark.sql.Column` - A new column of map type where each string in the original column is converted into a map. - Returns a column that evaluates to a map. + the column for calculating cumulative distribution. - Examples + See Also -------- - Example 1: Using default delimiters + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3",)], ["e"]) - >>> df.select(sf.str_to_map(df.e)).show(truncate=False) - +------------------------+ - |str_to_map(e, ,, :) | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - +------------------------+ - - Example 2: Using custom delimiters + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("cd", sf.cume_dist().over(w)).show() + +-----+---+ + |value| cd| + +-----+---+ + | 1|0.2| + | 2|0.4| + | 3|0.8| + | 3|0.8| + | 4|1.0| + +-----+---+ + """ + return _invoke_function("cume_dist") - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a=1;b=2;c=3",)], ["e"]) - >>> df.select(sf.str_to_map(df.e, sf.lit(";"), sf.lit("="))).show(truncate=False) - +------------------------+ - |str_to_map(e, ;, =) | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - +------------------------+ - Example 3: Using different delimiters for different rows +@_try_remote_functions +def percent_rank() -> Column: + """ + Window function: returns the relative rank (i.e. percentile) of rows within a window partition. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3",), ("d=4;e=5;f=6",)], ["e"]) - >>> df.select(sf.str_to_map(df.e, - ... sf.when(df.e.contains(";"), sf.lit(";")).otherwise(sf.lit(",")), - ... sf.when(df.e.contains("="), sf.lit("=")).otherwise(sf.lit(":"))).alias("str_to_map") - ... ).show(truncate=False) - +------------------------+ - |str_to_map | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - |{d -> 4, e -> 5, f -> 6}| - +------------------------+ + .. versionadded:: 1.6.0 - Example 4: Using a column of delimiters + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3", ","), ("d=4;e=5;f=6", ";")], ["e", "delim"]) - >>> df.select(sf.str_to_map(df.e, df.delim, sf.lit(":"))).show(truncate=False) - +---------------------------------------+ - |str_to_map(e, delim, :) | - +---------------------------------------+ - |{a -> 1, b -> 2, c -> 3} | - |{d=4 -> NULL, e=5 -> NULL, f=6 -> NULL}| - +---------------------------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + the column for calculating relative rank. - Example 5: Using a column of key/value delimiters + See Also + -------- + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3", ":"), ("d=4;e=5;f=6", "=")], ["e", "delim"]) - >>> df.select(sf.str_to_map(df.e, sf.lit(","), df.delim)).show(truncate=False) - +------------------------+ - |str_to_map(e, ,, delim) | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - |{d -> 4;e=5;f=6} | - +------------------------+ + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("pr", sf.percent_rank().over(w)).show() + +-----+---+ + |value| pr| + +-----+---+ + | 1|0.0| + | 1|0.0| + | 2|0.4| + | 3|0.6| + | 3|0.6| + | 4|1.0| + +-----+---+ """ - if pairDelim is None: - pairDelim = lit(",") - if keyValueDelim is None: - keyValueDelim = lit(":") - return _invoke_function_over_columns("str_to_map", text, pairDelim, keyValueDelim) - - -# ---------------------- Partition transform functions -------------------------------- + return _invoke_function("percent_rank") @_try_remote_functions -def years(col: "ColumnOrName") -> Column: +def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: """ - Partition transform function: A transform for timestamps and dates - to partition data into years. + Window function: returns the value that is `offset` rows before the current row, and + `default` if there is less than `offset` rows before the current row. For example, + an `offset` of one will return the previous row at any given point in the window partition. - .. versionadded:: 3.1.0 + This is equivalent to the LAG function in SQL. + + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. deprecated:: 4.0.0 - Use :func:`partitioning.years` instead. - Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target date or timestamp column to work on. + col : :class:`~pyspark.sql.Column` or column name + name of column or expression + offset : int, optional default 1 + number of row to extend + default : optional + default value Returns ------- :class:`~pyspark.sql.Column` - data partitioned by years. + value before current row based on `offset`. + + See Also + -------- + :meth:`pyspark.sql.functions.lead` Examples -------- - >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP - ... years("ts") - ... ).createOrReplace() + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ - Notes - ----- - This function can be used only in combination with - :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("previous_value", sf.lag("c2").over(w)).show() + +---+---+--------------+ + | c1| c2|previous_value| + +---+---+--------------+ + | a| 1| NULL| + | a| 2| 1| + | a| 3| 2| + | b| 2| NULL| + | b| 8| 2| + +---+---+--------------+ - """ - from pyspark.sql.functions import partitioning + >>> df.withColumn("previous_value", sf.lag("c2", 1, 0).over(w)).show() + +---+---+--------------+ + | c1| c2|previous_value| + +---+---+--------------+ + | a| 1| 0| + | a| 2| 1| + | a| 3| 2| + | b| 2| 0| + | b| 8| 2| + +---+---+--------------+ - warnings.warn("Deprecated in 4.0.0, use partitioning.years instead.", FutureWarning) + >>> df.withColumn("previous_value", sf.lag("c2", 2, -1).over(w)).show() + +---+---+--------------+ + | c1| c2|previous_value| + +---+---+--------------+ + | a| 1| -1| + | a| 2| -1| + | a| 3| 1| + | b| 2| -1| + | b| 8| -1| + +---+---+--------------+ + """ + from pyspark.sql.classic.column import _to_java_column - return partitioning.years(col) + return _invoke_function( + "lag", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) + ) @_try_remote_functions -def months(col: "ColumnOrName") -> Column: +def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: """ - Partition transform function: A transform for timestamps and dates - to partition data into months. + Window function: returns the value that is `offset` rows after the current row, and + `default` if there is less than `offset` rows after the current row. For example, + an `offset` of one will return the next row at any given point in the window partition. - .. versionadded:: 3.1.0 + This is equivalent to the LEAD function in SQL. + + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. deprecated:: 4.0.0 - Use :func:`partitioning.months` instead. - Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target date or timestamp column to work on. + col : :class:`~pyspark.sql.Column` or column name + name of column or expression + offset : int, optional default 1 + number of row to extend + default : optional + default value Returns ------- :class:`~pyspark.sql.Column` - data partitioned by months. + value after current row based on `offset`. + + See Also + -------- + :meth:`pyspark.sql.functions.lag` Examples -------- - >>> df.writeTo("catalog.db.table").partitionedBy( - ... months("ts") - ... ).createOrReplace() # doctest: +SKIP + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ - Notes - ----- - This function can be used only in combination with - :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("next_value", sf.lead("c2").over(w)).show() + +---+---+----------+ + | c1| c2|next_value| + +---+---+----------+ + | a| 1| 2| + | a| 2| 3| + | a| 3| NULL| + | b| 2| 8| + | b| 8| NULL| + +---+---+----------+ - """ - from pyspark.sql.functions import partitioning + >>> df.withColumn("next_value", sf.lead("c2", 1, 0).over(w)).show() + +---+---+----------+ + | c1| c2|next_value| + +---+---+----------+ + | a| 1| 2| + | a| 2| 3| + | a| 3| 0| + | b| 2| 8| + | b| 8| 0| + +---+---+----------+ - warnings.warn("Deprecated in 4.0.0, use partitioning.months instead.", FutureWarning) + >>> df.withColumn("next_value", sf.lead("c2", 2, -1).over(w)).show() + +---+---+----------+ + | c1| c2|next_value| + +---+---+----------+ + | a| 1| 3| + | a| 2| -1| + | a| 3| -1| + | b| 2| -1| + | b| 8| -1| + +---+---+----------+ + """ + from pyspark.sql.classic.column import _to_java_column - return partitioning.months(col) + return _invoke_function( + "lead", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) + ) @_try_remote_functions -def days(col: "ColumnOrName") -> Column: +def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = False) -> Column: """ - Partition transform function: A transform for timestamps and dates - to partition data into days. + Window function: returns the value that is the `offset`\\th row of the window frame + (counting from 1), and `null` if the size of window frame is less than `offset` rows. + + It will return the `offset`\\th non-null value it sees when `ignoreNulls` is set to + true. If all values are null, then null is returned. + + This is equivalent to the nth_value function in SQL. .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. deprecated:: 4.0.0 - Use :func:`partitioning.months` instead. - Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target date or timestamp column to work on. + col : :class:`~pyspark.sql.Column` or column name + name of column or expression + offset : int + number of row to use as the value + ignoreNulls : bool, optional + indicates the Nth value should skip null in the + determination of which row to use Returns ------- :class:`~pyspark.sql.Column` - data partitioned by days. + value of nth row. + + See Also + -------- + :meth:`pyspark.sql.functions.first_value` + :meth:`pyspark.sql.functions.last_value` Examples -------- - >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP - ... days("ts") - ... ).createOrReplace() + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ - Notes - ----- - This function can be used only in combination with - :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("nth_value", sf.nth_value("c2", 1).over(w)).show() + +---+---+---------+ + | c1| c2|nth_value| + +---+---+---------+ + | a| 1| 1| + | a| 2| 1| + | a| 3| 1| + | b| 2| 2| + | b| 8| 2| + +---+---+---------+ + >>> df.withColumn("nth_value", sf.nth_value("c2", 2).over(w)).show() + +---+---+---------+ + | c1| c2|nth_value| + +---+---+---------+ + | a| 1| NULL| + | a| 2| 2| + | a| 3| 2| + | b| 2| NULL| + | b| 8| 8| + +---+---+---------+ """ - from pyspark.sql.functions import partitioning - - warnings.warn("Deprecated in 4.0.0, use partitioning.days instead.", FutureWarning) + from pyspark.sql.classic.column import _to_java_column - return partitioning.days(col) + return _invoke_function( + "nth_value", _to_java_column(col), _enum_to_value(offset), _enum_to_value(ignoreNulls) + ) @_try_remote_functions -def hours(col: "ColumnOrName") -> Column: +def ntile(n: int) -> Column: """ - Partition transform function: A transform for timestamps - to partition data into hours. + Window function: returns the ntile group id (from 1 to `n` inclusive) + in an ordered window partition. For example, if `n` is 4, the first + quarter of the rows will get value 1, the second quarter will get 2, + the third quarter will get 3, and the last quarter will get 4. - .. versionadded:: 3.1.0 + This is equivalent to the NTILE function in SQL. + + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. deprecated:: 4.0.0 - Use :func:`partitioning.hours` instead. - Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target date or timestamp column to work on. + n : int + an integer Returns ------- :class:`~pyspark.sql.Column` - data partitioned by hours. + portioned group id. - Examples + See Also -------- - >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP - ... hours("ts") - ... ).createOrReplace() + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` - Notes - ----- - This function can be used only in combination with - :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("ntile", sf.ntile(2).over(w)).show() + +---+---+-----+ + | c1| c2|ntile| + +---+---+-----+ + | a| 1| 1| + | a| 2| 1| + | a| 3| 2| + | b| 2| 1| + | b| 8| 2| + +---+---+-----+ """ - from pyspark.sql.functions import partitioning + return _invoke_function("ntile", int(_enum_to_value(n))) - warnings.warn("Deprecated in 4.0.0, use partitioning.hours instead.", FutureWarning) - return partitioning.hours(col) +# ---------------------- Generator Functions ---------------------- @_try_remote_functions -def convert_timezone( - sourceTz: Optional[Column], targetTz: Column, sourceTs: "ColumnOrName" -) -> Column: +def explode(col: "ColumnOrName") -> Column: """ - Converts the timestamp without time zone `sourceTs` - from the `sourceTz` time zone to `targetTz`. + Returns a new row for each element in the given array or map. + Uses the default column name `col` for elements in the array and + `key` and `value` for elements in the map unless specified otherwise. - .. versionadded:: 3.5.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - sourceTz : :class:`~pyspark.sql.Column`, optional - The time zone for the input timestamp. If it is missed, - the current session time zone is used as the source time zone. - A column that evaluates to a string. - targetTz : :class:`~pyspark.sql.Column` - The time zone to which the input timestamp should be converted. - A column that evaluates to a string. - sourceTs : :class:`~pyspark.sql.Column` or column name - A timestamp without time zone. - A column that evaluates to a timestamp. + col : :class:`~pyspark.sql.Column` or column name + Target column to work on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a timestamp for converted time zone. - Returns a column that evaluates to a timestamp. + One row per array item or map key value. + Returns a column of the element type of the input array, or the key and value + columns of the input map. See Also -------- - :meth:`pyspark.sql.functions.current_timezone` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` + + Notes + ----- + Only one explode is allowed per SELECT clause. Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Exploding an array column - Example 1: Converts the timestamp without time zone `sourceTs`. + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.show() + +---+---------------+ + | i| a| + +---+---------------+ + | 1|[1, 2, 3, NULL]| + | 2| []| + | 3| NULL| + +---+---------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08 00:00:00',)], ['ts']) - >>> df.select( - ... '*', - ... sf.convert_timezone(None, sf.lit('Asia/Hong_Kong'), 'ts') - ... ).show() # doctest: +SKIP - +-------------------+--------------------------------------------------------+ - | ts|convert_timezone(current_timezone(), Asia/Hong_Kong, ts)| - +-------------------+--------------------------------------------------------+ - |2015-04-08 00:00:00| 2015-04-08 15:00:00| - +-------------------+--------------------------------------------------------+ + >>> df.select('*', sf.explode('a')).show() + +---+---------------+----+ + | i| a| col| + +---+---------------+----+ + | 1|[1, 2, 3, NULL]| 1| + | 1|[1, 2, 3, NULL]| 2| + | 1|[1, 2, 3, NULL]| 3| + | 1|[1, 2, 3, NULL]|NULL| + +---+---------------+----+ - Example 2: Converts the timestamp with time zone `sourceTs`. + Example 2: Exploding a map column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08 15:00:00',)], ['ts']) - >>> df.select( - ... '*', - ... sf.convert_timezone(sf.lit('Asia/Hong_Kong'), sf.lit('America/Los_Angeles'), df.ts) - ... ).show() - +-------------------+---------------------------------------------------------+ - | ts|convert_timezone(Asia/Hong_Kong, America/Los_Angeles, ts)| - +-------------------+---------------------------------------------------------+ - |2015-04-08 15:00:00| 2015-04-08 00:00:00| - +-------------------+---------------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.show(truncate=False) + +---+---------------------------+ + |i |m | + +---+---------------------------+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| + |2 |{} | + |3 |NULL | + +---+---------------------------+ - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> df.select('*', sf.explode('m')).show(truncate=False) + +---+---------------------------+---+-----+ + |i |m |key|value| + +---+---------------------------+---+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | + +---+---------------------------+---+-----+ + + Example 3: Exploding multiple array columns + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql('SELECT ARRAY(1,2) AS a1, ARRAY(3,4,5) AS a2') + >>> df.select( + ... '*', sf.explode('a1').alias('v1') + ... ).select('*', sf.explode('a2').alias('v2')).show() + +------+---------+---+---+ + | a1| a2| v1| v2| + +------+---------+---+---+ + |[1, 2]|[3, 4, 5]| 1| 3| + |[1, 2]|[3, 4, 5]| 1| 4| + |[1, 2]|[3, 4, 5]| 1| 5| + |[1, 2]|[3, 4, 5]| 2| 3| + |[1, 2]|[3, 4, 5]| 2| 4| + |[1, 2]|[3, 4, 5]| 2| 5| + +------+---------+---+---+ + + Example 4: Exploding an array of struct column + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select(sf.explode('a').alias("s")).select("s.*").show() + +---+---+ + | a| b| + +---+---+ + | 1| 2| + | 3| 4| + +---+---+ """ - if sourceTz is None: - return _invoke_function_over_columns("convert_timezone", targetTz, sourceTs) - else: - return _invoke_function_over_columns("convert_timezone", sourceTz, targetTz, sourceTs) + return _invoke_function_over_columns("explode", col) @_try_remote_functions -def make_dt_interval( - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, -) -> Column: +def posexplode(col: "ColumnOrName") -> Column: """ - Make DayTimeIntervalType duration from days, hours, mins and secs. + Returns a new row for each element with position in the given array or map. + Uses the default column name `pos` for position, and `col` for elements in the + array and `key` and `value` for elements in the map unless specified otherwise. - .. versionadded:: 3.5.0 + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - days : :class:`~pyspark.sql.Column` or column name, optional - The number of days, positive or negative. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The number of hours, positive or negative. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The number of minutes, positive or negative. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The number of seconds with the fractional part in microsecond precision. - A column that evaluates to a decimal. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a DayTimeIntervalType duration. - Returns a column that evaluates to an interval. + one row per array item or map key value including positions as a separate column. See Also -------- - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.make_ym_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` Examples -------- - Example 1: Make DayTimeIntervalType duration from days, hours, mins and secs. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval(df.day, df.hour, df.min, df.sec)).show(truncate=False) - +---+----+---+--------+------------------------------------------+ - |day|hour|min|sec |make_dt_interval(day, hour, min, sec) | - +---+----+---+--------+------------------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 12:30:01.001001' DAY TO SECOND| - +---+----+---+--------+------------------------------------------+ - - Example 2: Make DayTimeIntervalType duration from days, hours and mins. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval(df.day, 'hour', df.min)).show(truncate=False) - +---+----+---+--------+-----------------------------------+ - |day|hour|min|sec |make_dt_interval(day, hour, min, 0)| - +---+----+---+--------+-----------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 12:30:00' DAY TO SECOND| - +---+----+---+--------+-----------------------------------+ - - Example 3: Make DayTimeIntervalType duration from days and hours. + Example 1: Exploding an array column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval(df.day, df.hour)).show(truncate=False) - +---+----+---+--------+-----------------------------------+ - |day|hour|min|sec |make_dt_interval(day, hour, 0, 0) | - +---+----+---+--------+-----------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 12:00:00' DAY TO SECOND| - +---+----+---+--------+-----------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.show() + +---+---------------+ + | i| a| + +---+---------------+ + | 1|[1, 2, 3, NULL]| + | 2| []| + | 3| NULL| + +---+---------------+ - Example 4: Make DayTimeIntervalType duration from days. + >>> df.select('*', sf.posexplode('a')).show() + +---+---------------+---+----+ + | i| a|pos| col| + +---+---------------+---+----+ + | 1|[1, 2, 3, NULL]| 0| 1| + | 1|[1, 2, 3, NULL]| 1| 2| + | 1|[1, 2, 3, NULL]| 2| 3| + | 1|[1, 2, 3, NULL]| 3|NULL| + +---+---------------+---+----+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval('day')).show(truncate=False) - +---+----+---+--------+-----------------------------------+ - |day|hour|min|sec |make_dt_interval(day, 0, 0, 0) | - +---+----+---+--------+-----------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 00:00:00' DAY TO SECOND| - +---+----+---+--------+-----------------------------------+ + Example 2: Exploding a map column - Example 5: Make empty interval. + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.show(truncate=False) + +---+---------------------------+ + |i |m | + +---+---------------------------+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| + |2 |{} | + |3 |NULL | + +---+---------------------------+ - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_dt_interval()).show(truncate=False) - +-----------------------------------+ - |make_dt_interval(0, 0, 0, 0) | - +-----------------------------------+ - |INTERVAL '0 00:00:00' DAY TO SECOND| - +-----------------------------------+ + >>> df.select('*', sf.posexplode('m')).show(truncate=False) + +---+---------------------------+---+---+-----+ + |i |m |pos|key|value| + +---+---------------------------+---+---+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | + +---+---------------------------+---+---+-----+ """ - _days = lit(0) if days is None else days - _hours = lit(0) if hours is None else hours - _mins = lit(0) if mins is None else mins - _secs = lit(decimal.Decimal(0)) if secs is None else secs - return _invoke_function_over_columns("make_dt_interval", _days, _hours, _mins, _secs) + return _invoke_function_over_columns("posexplode", col) @_try_remote_functions -def try_make_interval( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - weeks: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, -) -> Column: +def inline(col: "ColumnOrName") -> Column: """ - This is a special version of `make_interval` that performs the same operation, but returns a - NULL value instead of raising an error if interval cannot be created. + Explodes an array of structs into a table. - .. versionadded:: 4.0.0 + This function takes an input column containing an array of structs and returns a + new column where each struct in the array is exploded into a separate row. + + .. versionadded:: 3.4.0 Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative. - A column that evaluates to an integer. - weeks : :class:`~pyspark.sql.Column` or column name, optional - The number of weeks, positive or negative. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The number of days, positive or negative. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The number of hours, positive or negative. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The number of minutes, positive or negative. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The number of seconds with the fractional part in microsecond precision. - A column that evaluates to a decimal. + col : :class:`~pyspark.sql.Column` or column name + Input column of values to explode. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an interval. - Returns a column that evaluates to an interval. + Generator expression with the inline exploded result. See Also -------- - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.make_dt_interval` - :meth:`pyspark.sql.functions.make_ym_interval` + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline_outer` Examples -------- - Example 1: Try make interval from years, months, weeks, days, hours, mins and secs. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_interval(df.year, df.month, 'week', df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +---------------------------------------------------------------+ - |try_make_interval(year, month, week, day, hour, min, sec) | - +---------------------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| - +---------------------------------------------------------------+ - - Example 2: Try make interval from years, months, weeks, days, hours and mins. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) - ... ).show(truncate=False) - +-------------------------------------------------------+ - |try_make_interval(year, month, week, day, hour, min, 0)| - +-------------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes | - +-------------------------------------------------------+ - - Example 3: Try make interval from years, months, weeks, days and hours. + Example 1: Using inline with a single struct array column >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour) - ... ).show(truncate=False) - +-----------------------------------------------------+ - |try_make_interval(year, month, week, day, hour, 0, 0)| - +-----------------------------------------------------+ - |100 years 11 months 8 days 12 hours | - +-----------------------------------------------------+ + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select('*', sf.inline(df.a)).show() + +----------------+---+---+ + | a| a| b| + +----------------+---+---+ + |[{1, 2}, {3, 4}]| 1| 2| + |[{1, 2}, {3, 4}]| 3| 4| + +----------------+---+---+ - Example 4: Try make interval from years, months, weeks and days. + Example 2: Using inline with a column name >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year, 'month', df.week, df.day)).show(truncate=False) - +--------------------------------------------------+ - |try_make_interval(year, month, week, day, 0, 0, 0)| - +--------------------------------------------------+ - |100 years 11 months 8 days | - +--------------------------------------------------+ + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select('*', sf.inline('a')).show() + +----------------+---+---+ + | a| a| b| + +----------------+---+---+ + |[{1, 2}, {3, 4}]| 1| 2| + |[{1, 2}, {3, 4}]| 3| 4| + +----------------+---+---+ - Example 5: Try make interval from years, months and weeks. + Example 3: Using inline with an alias >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year, 'month', df.week)).show(truncate=False) - +------------------------------------------------+ - |try_make_interval(year, month, week, 0, 0, 0, 0)| - +------------------------------------------------+ - |100 years 11 months 7 days | - +------------------------------------------------+ + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select('*', sf.inline('a').alias("c1", "c2")).show() + +----------------+---+---+ + | a| c1| c2| + +----------------+---+---+ + |[{1, 2}, {3, 4}]| 1| 2| + |[{1, 2}, {3, 4}]| 3| 4| + +----------------+---+---+ - Example 6: Try make interval from years and months. + Example 4: Using inline with multiple struct array columns >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year, 'month')).show(truncate=False) - +---------------------------------------------+ - |try_make_interval(year, month, 0, 0, 0, 0, 0)| - +---------------------------------------------+ - |100 years 11 months | - +---------------------------------------------+ + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a1, ARRAY(NAMED_STRUCT("c",5,"d",6), NAMED_STRUCT("c",7,"d",8)) AS a2') + >>> df.select( + ... '*', sf.inline('a1') + ... ).select('*', sf.inline('a2')).show() + +----------------+----------------+---+---+---+---+ + | a1| a2| a| b| c| d| + +----------------+----------------+---+---+---+---+ + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 5| 6| + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 7| 8| + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 5| 6| + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 7| 8| + +----------------+----------------+---+---+---+---+ - Example 7: Try make interval from years. + Example 5: Using inline with a nested struct array column >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year)).show(truncate=False) - +-----------------------------------------+ - |try_make_interval(year, 0, 0, 0, 0, 0, 0)| - +-----------------------------------------+ - |100 years | - +-----------------------------------------+ - - Example 8: Try make empty interval. + >>> df = spark.sql('SELECT NAMED_STRUCT("a",1,"b",2,"c",ARRAY(NAMED_STRUCT("c",3,"d",4), NAMED_STRUCT("c",5,"d",6))) AS s') + >>> df.select('*', sf.inline('s.c')).show(truncate=False) + +------------------------+---+---+ + |s |c |d | + +------------------------+---+---+ + |{1, 2, [{3, 4}, {5, 6}]}|3 |4 | + |{1, 2, [{3, 4}, {5, 6}]}|5 |6 | + +------------------------+---+---+ - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.try_make_interval()).show(truncate=False) - +--------------------------------------+ - |try_make_interval(0, 0, 0, 0, 0, 0, 0)| - +--------------------------------------+ - |0 seconds | - +--------------------------------------+ + Example 6: Using inline with a column containing: array continaing null, empty array and null - Example 9: Try make interval from years with overflow. + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') + >>> df.show(truncate=False) + +---+----------------------+ + |i |s | + +---+----------------------+ + |1 |[{1, 2}, NULL, {3, 4}]| + |2 |[] | + |3 |NULL | + +---+----------------------+ - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.try_make_interval(sf.lit(2147483647))).show(truncate=False) - +-----------------------------------------------+ - |try_make_interval(2147483647, 0, 0, 0, 0, 0, 0)| - +-----------------------------------------------+ - |NULL | - +-----------------------------------------------+ + >>> df.select('*', sf.inline('s')).show(truncate=False) + +---+----------------------+----+----+ + |i |s |a |b | + +---+----------------------+----+----+ + |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | + |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| + |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | + +---+----------------------+----+----+ """ - _years = lit(0) if years is None else years - _months = lit(0) if months is None else months - _weeks = lit(0) if weeks is None else weeks - _days = lit(0) if days is None else days - _hours = lit(0) if hours is None else hours - _mins = lit(0) if mins is None else mins - _secs = lit(decimal.Decimal(0)) if secs is None else secs - return _invoke_function_over_columns( - "try_make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs - ) + return _invoke_function_over_columns("inline", col) @_try_remote_functions -def make_interval( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - weeks: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, -) -> Column: +def explode_outer(col: "ColumnOrName") -> Column: """ - Make interval from years, months, weeks, days, hours, mins and secs. + Returns a new row for each element in the given array or map. + Unlike explode, if the array/map is null or empty then null is produced. + Uses the default column name `col` for elements in the array and + `key` and `value` for elements in the map unless specified otherwise. - .. versionadded:: 3.5.0 + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative. - A column that evaluates to an integer. - weeks : :class:`~pyspark.sql.Column` or column name, optional - The number of weeks, positive or negative. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The number of days, positive or negative. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The number of hours, positive or negative. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The number of minutes, positive or negative. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The number of seconds with the fractional part in microsecond precision. - A column that evaluates to a decimal. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an interval. - Returns a column that evaluates to an interval. + one row per array item or map key value. + Returns a column of the element type of the input array, or the key and value + columns of the input map. See Also -------- - :meth:`pyspark.sql.functions.make_dt_interval` - :meth:`pyspark.sql.functions.make_ym_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` Examples -------- - Example 1: Make interval from years, months, weeks, days, hours, mins and secs. + Example 1: Using an array column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +---------------------------------------------------------------+ - |make_interval(year, month, week, day, hour, min, sec) | - +---------------------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| - +---------------------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.select('*', sf.explode_outer('a')).show() + +---+---------------+----+ + | i| a| col| + +---+---------------+----+ + | 1|[1, 2, 3, NULL]| 1| + | 1|[1, 2, 3, NULL]| 2| + | 1|[1, 2, 3, NULL]| 3| + | 1|[1, 2, 3, NULL]|NULL| + | 2| []|NULL| + | 3| NULL|NULL| + +---+---------------+----+ - Example 2: Make interval from years, months, weeks, days, hours and mins. + Example 2: Using a map column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) - ... ).show(truncate=False) - +---------------------------------------------------+ - |make_interval(year, month, week, day, hour, min, 0)| - +---------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes | - +---------------------------------------------------+ - - Example 3: Make interval from years, months, weeks, days and hours. + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.select('*', sf.explode_outer('m')).show(truncate=False) + +---+---------------------------+----+-----+ + |i |m |key |value| + +---+---------------------------+----+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | + |2 |{} |NULL|NULL | + |3 |NULL |NULL|NULL | + +---+---------------------------+----+-----+ + """ + return _invoke_function_over_columns("explode_outer", col) - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour) - ... ).show(truncate=False) - +-------------------------------------------------+ - |make_interval(year, month, week, day, hour, 0, 0)| - +-------------------------------------------------+ - |100 years 11 months 8 days 12 hours | - +-------------------------------------------------+ - Example 4: Make interval from years, months, weeks and days. +@_try_remote_functions +def posexplode_outer(col: "ColumnOrName") -> Column: + """ + Returns a new row for each element with position in the given array or map. + Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced. + Uses the default column name `pos` for position, and `col` for elements in the + array and `key` and `value` for elements in the map unless specified otherwise. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year, df.month, 'week', df.day)).show(truncate=False) - +----------------------------------------------+ - |make_interval(year, month, week, day, 0, 0, 0)| - +----------------------------------------------+ - |100 years 11 months 8 days | - +----------------------------------------------+ + .. versionadded:: 2.3.0 - Example 5: Make interval from years, months and weeks. + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year, df.month, 'week')).show(truncate=False) - +--------------------------------------------+ - |make_interval(year, month, week, 0, 0, 0, 0)| - +--------------------------------------------+ - |100 years 11 months 7 days | - +--------------------------------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. - Example 6: Make interval from years and months. + Returns + ------- + :class:`~pyspark.sql.Column` + one row per array item or map key value including positions as a separate column. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year, df.month)).show(truncate=False) - +-----------------------------------------+ - |make_interval(year, month, 0, 0, 0, 0, 0)| - +-----------------------------------------+ - |100 years 11 months | - +-----------------------------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` - Example 7: Make interval from years. + Examples + -------- + Example 1: Using an array column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year)).show(truncate=False) - +-------------------------------------+ - |make_interval(year, 0, 0, 0, 0, 0, 0)| - +-------------------------------------+ - |100 years | - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.select('*', sf.posexplode_outer('a')).show() + +---+---------------+----+----+ + | i| a| pos| col| + +---+---------------+----+----+ + | 1|[1, 2, 3, NULL]| 0| 1| + | 1|[1, 2, 3, NULL]| 1| 2| + | 1|[1, 2, 3, NULL]| 2| 3| + | 1|[1, 2, 3, NULL]| 3|NULL| + | 2| []|NULL|NULL| + | 3| NULL|NULL|NULL| + +---+---------------+----+----+ - Example 8: Make empty interval. + Example 2: Using a map column - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_interval()).show(truncate=False) - +----------------------------------+ - |make_interval(0, 0, 0, 0, 0, 0, 0)| - +----------------------------------+ - |0 seconds | - +----------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.select('*', sf.posexplode_outer('m')).show(truncate=False) + +---+---------------------------+----+----+-----+ + |i |m |pos |key |value| + +---+---------------------------+----+----+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | + |2 |{} |NULL|NULL|NULL | + |3 |NULL |NULL|NULL|NULL | + +---+---------------------------+----+----+-----+ """ - _years = lit(0) if years is None else years - _months = lit(0) if months is None else months - _weeks = lit(0) if weeks is None else weeks - _days = lit(0) if days is None else days - _hours = lit(0) if hours is None else hours - _mins = lit(0) if mins is None else mins - _secs = lit(decimal.Decimal(0)) if secs is None else secs - return _invoke_function_over_columns( - "make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs - ) + return _invoke_function_over_columns("posexplode_outer", col) @_try_remote_functions -def make_time(hour: "ColumnOrName", minute: "ColumnOrName", second: "ColumnOrName") -> Column: +def inline_outer(col: "ColumnOrName") -> Column: """ - Create time from hour, minute and second fields. For invalid inputs it will throw an error. + Explodes an array of structs into a table. + Unlike inline, if the array is null or empty then null is produced for each nested column. - .. versionadded:: 4.1.0 + .. versionadded:: 3.4.0 Parameters ---------- - hour : :class:`~pyspark.sql.Column` or column name - The hour to represent, from 0 to 23. - A column that evaluates to an integer. - minute : :class:`~pyspark.sql.Column` or column name - The minute to represent, from 0 to 59. - A column that evaluates to an integer. - second : :class:`~pyspark.sql.Column` or column name - The second to represent, from 0 to 59.999999. - A column that evaluates to a decimal. + col : :class:`~pyspark.sql.Column` or column name + input column of values to explode. Returns ------- :class:`~pyspark.sql.Column` - A column representing the created time. - Returns a column that evaluates to a time. + generator expression with the inline exploded result. + + See Also + -------- + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + + Notes + ----- + Supports Spark Connect. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(6, 30, 45.887)], ["hour", "minute", "second"]) - >>> df.select(sf.make_time("hour", "minute", "second").alias("time")).show() - +------------+ - | time| - +------------+ - |06:30:45.887| - +------------+ + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') + >>> df.printSchema() + root + |-- i: integer (nullable = false) + |-- s: array (nullable = true) + | |-- element: struct (containsNull = true) + | | |-- a: integer (nullable = false) + | | |-- b: integer (nullable = false) + + >>> df.select('*', sf.inline_outer('s')).show(truncate=False) + +---+----------------------+----+----+ + |i |s |a |b | + +---+----------------------+----+----+ + |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | + |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| + |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | + |2 |[] |NULL|NULL| + |3 |NULL |NULL|NULL| + +---+----------------------+----+----+ """ - return _invoke_function_over_columns("make_time", hour, minute, second) + return _invoke_function_over_columns("inline_outer", col) @_try_remote_functions -def time_from_seconds(col: "ColumnOrName") -> Column: +def stack(*cols: "ColumnOrName") -> Column: """ - Creates a TIME value from seconds since midnight (supports fractional seconds). + Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default + unless specified otherwise. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Seconds since midnight (0 to 86399.999999). - A column that evaluates to a numeric. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a literal int for the number of rows to be separated, + and the remaining are input elements to be separated. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(52200.5,)], ['seconds']) - >>> df.select(sf.time_from_seconds('seconds')).show() - +--------------------------+ - |time_from_seconds(seconds)| - +--------------------------+ - | 14:30:00.5| - +--------------------------+ + >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) + >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c')).show() + +---+---+---+----+----+ + | a| b| c|col0|col1| + +---+---+---+----+----+ + | 1| 2| 3| 1| 2| + | 1| 2| 3| 3|NULL| + +---+---+---+----+----+ + + >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c').alias('x', 'y')).show() + +---+---+---+---+----+ + | a| b| c| x| y| + +---+---+---+---+----+ + | 1| 2| 3| 1| 2| + | 1| 2| 3| 3|NULL| + +---+---+---+---+----+ + + >>> df.select('*', sf.stack(sf.lit(3), df.a, df.b, 'c')).show() + +---+---+---+----+ + | a| b| c|col0| + +---+---+---+----+ + | 1| 2| 3| 1| + | 1| 2| 3| 2| + | 1| 2| 3| 3| + +---+---+---+----+ + + >>> df.select('*', sf.stack(sf.lit(4), df.a, df.b, 'c')).show() + +---+---+---+----+ + | a| b| c|col0| + +---+---+---+----+ + | 1| 2| 3| 1| + | 1| 2| 3| 2| + | 1| 2| 3| 3| + | 1| 2| 3|NULL| + +---+---+---+----+ """ - return _invoke_function_over_columns("time_from_seconds", col) + return _invoke_function_over_seq_of_columns("stack", cols) + + +# ---------------------- Partition Transformation Functions ---------------------- @_try_remote_functions -def time_from_millis(col: "ColumnOrName") -> Column: +def years(col: "ColumnOrName") -> Column: """ - Creates a TIME value from milliseconds since midnight. + Partition transform function: A transform for timestamps and dates + to partition data into years. - .. versionadded:: 4.2.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 4.0.0 + Use :func:`partitioning.years` instead. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Milliseconds since midnight (0 to 86399999). - A column that evaluates to an integral. + col : :class:`~pyspark.sql.Column` or str + target date or timestamp column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + data partitioned by years. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(52200500,)], ['millis']) - >>> df.select(sf.time_from_millis('millis')).show() - +------------------------+ - |time_from_millis(millis)| - +------------------------+ - | 14:30:00.5| - +------------------------+ + >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP + ... years("ts") + ... ).createOrReplace() + + Notes + ----- + This function can be used only in combination with + :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` + method of the `DataFrameWriterV2`. + """ - return _invoke_function_over_columns("time_from_millis", col) + from pyspark.sql.functions import partitioning + + warnings.warn("Deprecated in 4.0.0, use partitioning.years instead.", FutureWarning) + + return partitioning.years(col) @_try_remote_functions -def time_from_micros(col: "ColumnOrName") -> Column: +def months(col: "ColumnOrName") -> Column: """ - Creates a TIME value from microseconds since midnight. + Partition transform function: A transform for timestamps and dates + to partition data into months. - .. versionadded:: 4.2.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 4.0.0 + Use :func:`partitioning.months` instead. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Microseconds since midnight (0 to 86399999999). - A column that evaluates to an integral. + col : :class:`~pyspark.sql.Column` or str + target date or timestamp column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + data partitioned by months. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(52200500000,)], ['micros']) - >>> df.select(sf.time_from_micros('micros')).show() - +------------------------+ - |time_from_micros(micros)| - +------------------------+ - | 14:30:00.5| - +------------------------+ + >>> df.writeTo("catalog.db.table").partitionedBy( + ... months("ts") + ... ).createOrReplace() # doctest: +SKIP + + Notes + ----- + This function can be used only in combination with + :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` + method of the `DataFrameWriterV2`. + """ - return _invoke_function_over_columns("time_from_micros", col) + from pyspark.sql.functions import partitioning + + warnings.warn("Deprecated in 4.0.0, use partitioning.months instead.", FutureWarning) + + return partitioning.months(col) @_try_remote_functions -def time_to_seconds(col: "ColumnOrName") -> Column: +def days(col: "ColumnOrName") -> Column: """ - Extracts seconds from TIME value (returns DECIMAL to preserve fractional seconds). + Partition transform function: A transform for timestamps and dates + to partition data into days. - .. versionadded:: 4.2.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 4.0.0 + Use :func:`partitioning.months` instead. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - TIME value to convert. + col : :class:`~pyspark.sql.Column` or str + target date or timestamp column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + data partitioned by days. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") - >>> df.select(sf.time_to_seconds('time')).show() - +---------------------+ - |time_to_seconds(time)| - +---------------------+ - | 52200.500000| - +---------------------+ + >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP + ... days("ts") + ... ).createOrReplace() + + Notes + ----- + This function can be used only in combination with + :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` + method of the `DataFrameWriterV2`. + """ - return _invoke_function_over_columns("time_to_seconds", col) + from pyspark.sql.functions import partitioning + + warnings.warn("Deprecated in 4.0.0, use partitioning.days instead.", FutureWarning) + + return partitioning.days(col) @_try_remote_functions -def time_to_millis(col: "ColumnOrName") -> Column: +def hours(col: "ColumnOrName") -> Column: """ - Extracts milliseconds from TIME value. + Partition transform function: A transform for timestamps + to partition data into hours. - .. versionadded:: 4.2.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 4.0.0 + Use :func:`partitioning.hours` instead. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - TIME value to convert. + col : :class:`~pyspark.sql.Column` or str + target date or timestamp column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + data partitioned by hours. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") - >>> df.select(sf.time_to_millis('time')).show() - +--------------------+ - |time_to_millis(time)| - +--------------------+ - | 52200500| - +--------------------+ + >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP + ... hours("ts") + ... ).createOrReplace() + + Notes + ----- + This function can be used only in combination with + :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` + method of the `DataFrameWriterV2`. + """ - return _invoke_function_over_columns("time_to_millis", col) + from pyspark.sql.functions import partitioning + + warnings.warn("Deprecated in 4.0.0, use partitioning.hours instead.", FutureWarning) + + return partitioning.hours(col) @_try_remote_functions -def time_to_micros(col: "ColumnOrName") -> Column: +def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column: """ - Extracts microseconds from TIME value. + Partition transform function: A transform for any type that partitions + by a hash of the input column. - .. versionadded:: 4.2.0 + .. versionadded:: 3.1.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - TIME value to convert. + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 4.0.0 + Use :func:`partitioning.bucket` instead. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") - >>> df.select(sf.time_to_micros('time')).show() - +--------------------+ - |time_to_micros(time)| - +--------------------+ - | 52200500000| - +--------------------+ + >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP + ... bucket(42, "ts") + ... ).createOrReplace() + + Parameters + ---------- + numBuckets : :class:`~pyspark.sql.Column` or int + the number of buckets + col : :class:`~pyspark.sql.Column` or str + target date or timestamp column to work on. + + Returns + ------- + :class:`~pyspark.sql.Column` + data partitioned by given columns. + + Notes + ----- + This function can be used only in combination with + :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` + method of the `DataFrameWriterV2`. + """ - return _invoke_function_over_columns("time_to_micros", col) + from pyspark.sql.functions import partitioning + warnings.warn("Deprecated in 4.0.0, use partitioning.bucket instead.", FutureWarning) -def _ensure_column_or_name(arg: Optional[Any]) -> "ColumnOrName": - if not isinstance(arg, (Column, str)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "arg", - "arg_type": type(arg).__name__, - }, - ) - return arg + return partitioning.bucket(numBuckets, col) -@overload -def make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... - - -@overload -def make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: "ColumnOrName", -) -> Column: ... - - -@overload -def make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... - - -@overload -def make_timestamp( - *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" -) -> Column: ... +# ---------------------- CSV Functions ---------------------- @_try_remote_functions -def make_timestamp( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - timezone: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, -) -> Column: +def schema_of_csv(csv: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: """ - Create timestamp from years, months, days, hours, mins, secs, and (optional) timezone fields. - Alternatively, create timestamp from date, time, and (optional) timezone fields. - The result data type is consistent with the value of configuration `spark.sql.timestampType`. - If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL - on invalid inputs. Otherwise, it will throw an error instead. + CSV Function: Parses a CSV string and infers its schema in DDL format. - .. versionadded:: 3.5.0 + .. versionadded:: 3.0.0 - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or column name, optional - The time zone identifier. For example, CET, UTC, and etc. + csv : :class:`~pyspark.sql.Column` or str A column that evaluates to a string. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. + A CSV string or a foldable string column containing a CSV string. + options : dict, optional + Options to control parsing. Accepts the same options as the CSV datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a timestamp. - Returns a column that evaluates to a timestamp. - - See Also - -------- - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + A string representation of a :class:`StructType` parsed from the given CSV. + Returns a column that evaluates to a string. Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Inferring the schema of a CSV string with different data types - Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. + >>> from pyspark.sql import functions as sf + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('1|a|true'), {'sep':'|'})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv(1|a|true) | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') - ... ).show(truncate=False) - +----------------------------------------------------+ - |make_timestamp(year, month, day, hour, min, sec, tz)| - +----------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +----------------------------------------------------+ + Example 2: Inferring the schema of a CSV string with missing values - Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). + >>> from pyspark.sql import functions as sf + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('1||true'), {'sep':'|'})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv(1||true) | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +------------------------------------------------+ - |make_timestamp(year, month, day, hour, min, sec)| - +------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +------------------------------------------------+ + Example 3: Inferring the schema of a CSV string with a different delimiter - Example 3: Make timestamp from date, time, and timezone. + >>> from pyspark.sql import functions as sf + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('1;a;true'), {'sep':';'})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv(1;a;true) | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time"), - ... sf.lit("CET").alias("tz") - ... ) - >>> df.select( - ... sf.make_timestamp(date=df.date, time=df.time, timezone=df.tz) - ... ).show(truncate=False) - +------------------------------+ - |make_timestamp(date, time, tz)| - +------------------------------+ - |2014-12-27 21:30:45.887 | - +------------------------------+ + Example 4: Inferring the schema of a CSV string with quoted fields - Example 4: Make timestamp from date and time (without timezone). + >>> from pyspark.sql import functions as sf + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('"1","a","true"'), {'sep':','})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv("1","a","true") | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ + """ + from pyspark.sql.classic.column import _to_java_column - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time") - ... ) - >>> df.select(sf.make_timestamp(date=df.date, time=df.time)).show(truncate=False) - +--------------------------+ - |make_timestamp(date, time)| - +--------------------------+ - |2014-12-28 06:30:45.887 | - +--------------------------+ + csv = _enum_to_value(csv) + if not isinstance(csv, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "csv", + "arg_type": type(csv).__name__, + }, + ) - >>> spark.conf.unset("spark.sql.session.timeZone") + return _invoke_function("schema_of_csv", _to_java_column(lit(csv)), _options_to_str(options)) + + +@_try_remote_functions +def to_csv(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) - else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "make_timestamp", - _ensure_column_or_name(date), - _ensure_column_or_name(time), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) + CSV Function: Converts a column containing a :class:`StructType` into a CSV string. + Throws an exception, in the case of an unsupported type. + .. versionadded:: 3.0.0 -@overload -def try_make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... + .. versionchanged:: 3.4.0 + Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a struct, array, map, or variant. + Name of column containing a struct. + options: dict, optional + Options to control converting. Accepts the same options as the CSV datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. -@overload -def try_make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: "ColumnOrName", -) -> Column: ... + .. # noqa + + Returns + ------- + :class:`~pyspark.sql.Column` + A CSV string converted from the given :class:`StructType`. + Returns a column that evaluates to a string. + Examples + -------- + Example 1: Converting a simple StructType to a CSV string -@overload -def try_make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... + >>> from pyspark.sql import Row, functions as sf + >>> data = [(1, Row(age=2, name='Alice'))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_csv(df.value)).show() + +-------------+ + |to_csv(value)| + +-------------+ + | 2,Alice| + +-------------+ + Example 2: Converting a complex StructType to a CSV string -@overload -def try_make_timestamp( - *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" -) -> Column: ... + >>> from pyspark.sql import Row, functions as sf + >>> data = [(1, Row(age=2, name='Alice', scores=[100, 200, 300]))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_csv(df.value)).show(truncate=False) + +-------------------------+ + |to_csv(value) | + +-------------------------+ + |2,Alice,"[100, 200, 300]"| + +-------------------------+ + + Example 3: Converting a StructType with null values to a CSV string + + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import StructType, StructField, IntegerType, StringType + >>> data = [(1, Row(age=None, name='Alice'))] + >>> schema = StructType([ + ... StructField("key", IntegerType(), True), + ... StructField("value", StructType([ + ... StructField("age", IntegerType(), True), + ... StructField("name", StringType(), True) + ... ]), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.to_csv(df.value)).show() + +-------------+ + |to_csv(value)| + +-------------+ + | ,Alice| + +-------------+ + + Example 4: Converting a StructType with different data types to a CSV string + + >>> from pyspark.sql import Row, functions as sf + >>> data = [(1, Row(age=2, name='Alice', isStudent=True))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_csv(df.value)).show() + +-------------+ + |to_csv(value)| + +-------------+ + | 2,Alice,true| + +-------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("to_csv", _to_java_column(col), _options_to_str(options)) @_try_remote_functions -def try_make_timestamp( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - timezone: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, +def from_csv( + col: "ColumnOrName", + schema: Union[Column, str], + options: Optional[Mapping[str, str]] = None, ) -> Column: """ - Try to create timestamp from years, months, days, hours, mins, secs and (optional) timezone - fields. Alternatively, try to create timestamp from date, time, and (optional) timezone fields. - The result data type is consistent with the value of configuration `spark.sql.timestampType`. - The function returns NULL on invalid inputs. + CSV Function: Parses a column containing a CSV string into a row with the specified schema. + Returns `null` if the string cannot be parsed. - .. versionadded:: 4.0.0 + .. versionadded:: 3.0.0 - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or column name, optional - The time zone identifier. For example, CET, UTC, and etc. + col : :class:`~pyspark.sql.Column` or str A column that evaluates to a string. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. + A column or column name in CSV format. + schema : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string, or a DDL-formatted type string, or a DataType. + A column, or Python string literal with schema in DDL format, to use when parsing the CSV column. + options : dict, optional + Options to control parsing. Accepts the same options as the CSV datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a timestamp or NULL in case of an error. - Returns a column that evaluates to a timestamp. - - See Also - -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + A column of parsed CSV values. + Returns a column that evaluates to a struct. Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Parsing a simple CSV string - Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. + >>> from pyspark.sql import functions as sf + >>> data = [("1,2,3",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, 3}| + +---------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') - ... ).show(truncate=False) - +----------------------------------------------------+ - |try_make_timestamp(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +----------------------------------------------------+ + Example 2: Using schema_of_csv to infer the schema - Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). + >>> from pyspark.sql import functions as sf + >>> data = [("1,2,3",)] + >>> value = data[0][0] + >>> df.select(sf.from_csv(df.value, sf.schema_of_csv(value))).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, 3}| + +---------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |try_make_timestamp(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +----------------------------------------------------+ + Example 3: Ignoring leading white space in the CSV string - Example 3: Make timestamp with invalid input. + >>> from pyspark.sql import functions as sf + >>> data = [(" abc",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> options = {'ignoreLeadingWhiteSpace': True} + >>> df.select(sf.from_csv(df.value, "s string", options)).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {abc}| + +---------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |try_make_timestamp(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |NULL | - +----------------------------------------------------+ + Example 4: Parsing a CSV string with a missing value - Example 4: Make timestamp from date, time, and timezone. + >>> from pyspark.sql import functions as sf + >>> data = [("1,2,",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, NULL}| + +---------------+ - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time"), - ... sf.lit("CET").alias("tz") - ... ) - >>> df.select( - ... sf.try_make_timestamp(date=df.date, time=df.time, timezone=df.tz) - ... ).show(truncate=False) - +----------------------------------+ - |try_make_timestamp(date, time, tz)| - +----------------------------------+ - |2014-12-27 21:30:45.887 | - +----------------------------------+ + Example 5: Parsing a CSV string with a different delimiter - Example 5: Make timestamp from date and time (without timezone). + >>> from pyspark.sql import functions as sf + >>> data = [("1;2;3",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> options = {'delimiter': ';'} + >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT", options)).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, 3}| + +---------------+ + """ + from pyspark.sql.classic.column import _to_java_column - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time") - ... ) - >>> df.select(sf.try_make_timestamp(date=df.date, time=df.time)).show(truncate=False) - +------------------------------+ - |try_make_timestamp(date, time)| - +------------------------------+ - |2014-12-28 06:30:45.887 | - +------------------------------+ + if not isinstance(schema, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "schema", + "arg_type": type(schema).__name__, + }, + ) - >>> spark.conf.unset("spark.sql.session.timeZone") + return _invoke_function( + "from_csv", _to_java_column(col), _to_java_column(lit(schema)), _options_to_str(options) + ) + + +# ---------------------- JSON Functions ---------------------- + + +@_try_remote_functions +def get_json_object(col: "ColumnOrName", path: str) -> Column: """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "try_make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "try_make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) - else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "try_make_timestamp", - _ensure_column_or_name(date), - _ensure_column_or_name(time), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "try_make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) + Extracts json object from a json string based on json `path` specified, and returns json string + of the extracted json object. It will return null if the input json string is invalid. + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + string column in json format. + A column that evaluates to a string. + path : str + path to the json object to extract. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + string representation of given JSON object value. + Returns a column that evaluates to a string. + + Examples + -------- + Example 1: Extract a json object from json string + + >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] + >>> df = spark.createDataFrame(data, ("key", "jstring")) + >>> df.select(df.key, + ... get_json_object(df.jstring, '$.f1').alias("c0"), + ... get_json_object(df.jstring, '$.f2').alias("c1") + ... ).show() + +---+-------+------+ + |key| c0| c1| + +---+-------+------+ + | 1| value1|value2| + | 2|value12| NULL| + +---+-------+------+ + + Example 2: Extract a json object from json array + + >>> data = [ + ... ("1", '''[{"f1": "value1"},{"f1": "value2"}]'''), + ... ("2", '''[{"f1": "value12"},{"f2": "value13"}]''') + ... ] + >>> df = spark.createDataFrame(data, ("key", "jarray")) + >>> df.select(df.key, + ... get_json_object(df.jarray, '$[0].f1').alias("c0"), + ... get_json_object(df.jarray, '$[1].f2').alias("c1") + ... ).show() + +---+-------+-------+ + |key| c0| c1| + +---+-------+-------+ + | 1| value1| NULL| + | 2|value12|value13| + +---+-------+-------+ + + >>> df.select(df.key, + ... get_json_object(df.jarray, '$[*].f1').alias("c0"), + ... get_json_object(df.jarray, '$[*].f2').alias("c1") + ... ).show() + +---+-------------------+---------+ + |key| c0| c1| + +---+-------------------+---------+ + | 1|["value1","value2"]| NULL| + | 2| "value12"|"value13"| + +---+-------------------+---------+ + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("get_json_object", _to_java_column(col), _enum_to_value(path)) @_try_remote_functions -def make_timestamp_ltz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: Optional["ColumnOrName"] = None, +def json_tuple(col: "ColumnOrName", *fields: str) -> Column: + """Creates a new row for a json column according to the given field names. + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + string column in json format + A column that evaluates to a string. + fields : str + a field or fields to extract + Each a column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new row for each given field value from json object + Returns a column that evaluates to a string. + + Examples + -------- + >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] + >>> df = spark.createDataFrame(data, ("key", "jstring")) + >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect() + [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)] + """ + from pyspark.sql.classic.column import _to_java_column, _to_seq + + if len(fields) == 0: + raise PySparkValueError( + errorClass="CANNOT_BE_EMPTY", + messageParameters={"item": "field"}, + ) + sc = _get_active_spark_context() + return _invoke_function("json_tuple", _to_java_column(col), _to_seq(sc, fields)) + + +@_try_remote_functions +def from_json( + col: "ColumnOrName", + schema: Union[ArrayType, StructType, MapType, Column, str], + options: Optional[Mapping[str, str]] = None, ) -> Column: """ - Create the current timestamp with local time zone from years, months, days, hours, mins, - secs and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, - the function returns NULL on invalid inputs. Otherwise, it will throw an error instead. + Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType` + as keys type, :class:`StructType` or :class:`ArrayType` with + the specified schema. Returns `null`, in the case of an unparsable string. - .. versionadded:: 3.5.0 + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or str - The year to represent, from 1 to 9999. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or str - The month-of-year to represent, from 1 (January) to 12 (December). - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or str - The day-of-month to represent, from 1 to 31. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or str - The hour-of-day to represent, from 0 to 23. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or str - The minute-of-hour to represent, from 0 to 59. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or str - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13 , or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or str, optional - The time zone identifier. For example, CET, UTC and etc. + col : :class:`~pyspark.sql.Column` or str A column that evaluates to a string. + a column or column name in JSON format + schema : :class:`StructType`, :class:`ArrayType`, :class:`MapType`, or str + a StructType, ArrayType of StructType, MapType, or Python string literal with a DDL-formatted string + A column that evaluates to a string, or a DDL-formatted type string, or a DataType. + to use when parsing the json column + options : dict, optional + options to control parsing. accepts the same options as the json datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a current timestamp. - Returns a column that evaluates to a timestamp. + a new column of complex type from given JSON object. + Returns a column that evaluates to a struct, array, or map. - See Also + Examples -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + Example 1: Parsing JSON with a specified schema + + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql.types import StructType, StructField, IntegerType + >>> schema = StructType([StructField("a", IntegerType())]) + >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, schema).alias("json")).show() + +----+ + |json| + +----+ + | {1}| + +----+ + + Example 2: Parsing JSON with a DDL-formatted string. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, "a INT").alias("json")).show() + +----+ + |json| + +----+ + | {1}| + +----+ + + Example 3: Parsing JSON into a MapType + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, "MAP").alias("json")).show() + +--------+ + | json| + +--------+ + |{a -> 1}| + +--------+ + + Example 4: Parsing JSON into an ArrayType of StructType + + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql.types import ArrayType, StructType, StructField, IntegerType + >>> schema = ArrayType(StructType([StructField("a", IntegerType())])) + >>> df = spark.createDataFrame([(1, '''[{"a": 1}]''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, schema).alias("json")).show() + +-----+ + | json| + +-----+ + |[{1}]| + +-----+ + + Example 5: Parsing JSON into an ArrayType + + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType + >>> schema = ArrayType(IntegerType()) + >>> df = spark.createDataFrame([(1, '''[1, 2, 3]''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, schema).alias("json")).show() + +---------+ + | json| + +---------+ + |[1, 2, 3]| + +---------+ + + Example 6: Parsing JSON with specified options + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, '''{a:123}'''), (2, '''{"a":456}''')], ("key", "value")) + >>> parsed1 = sf.from_json(df.value, "a INT") + >>> parsed2 = sf.from_json(df.value, "a INT", {"allowUnquotedFieldNames": "true"}) + >>> df.select("value", parsed1, parsed2).show() + +---------+----------------+----------------+ + | value|from_json(value)|from_json(value)| + +---------+----------------+----------------+ + | {a:123}| {NULL}| {123}| + |{"a":456}| {456}| {456}| + +---------+----------------+----------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + if isinstance(schema, DataType): + schema = schema.json() + elif isinstance(schema, Column): + schema = _to_java_column(schema) + return _invoke_function("from_json", _to_java_column(col), schema, _options_to_str(options)) + + +@_try_remote_functions +def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: + """ + Converts a column containing a :class:`StructType`, :class:`ArrayType`, :class:`MapType` + or a :class:`VariantType` into a JSON string. Throws an exception, in the case of an unsupported type. + + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + name of column containing a struct, an array, a map, or a variant object. + A column that evaluates to a struct, array, map, or variant. + options : dict, optional + options to control converting. accepts the same options as the JSON datasource. + See `Data Source Option `_ + for the version you use. + Additionally the function supports the `pretty` option which enables + A dict of options. Each key and value is a string. + pretty JSON generation. + + .. # noqa + + Returns + ------- + :class:`~pyspark.sql.Column` + JSON object as string column. + Returns a column that evaluates to a string. Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Converting a StructType column to JSON - Example 1: Make the current timestamp from years, months, days, hours, mins and secs. + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql import Row + >>> data = [(1, Row(age=2, name='Alice'))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +------------------------+ + |json | + +------------------------+ + |{"age":2,"name":"Alice"}| + +------------------------+ + + Example 2: Converting an ArrayType column to JSON >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec, 'tz') - ... ).show(truncate=False) - +--------------------------------------------------------+ - |make_timestamp_ltz(year, month, day, hour, min, sec, tz)| - +--------------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +--------------------------------------------------------+ + >>> from pyspark.sql import Row + >>> data = [(1, [Row(age=2, name='Alice'), Row(age=3, name='Bob')])] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +-------------------------------------------------+ + |json | + +-------------------------------------------------+ + |[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]| + +-------------------------------------------------+ - Example 2: Make the current timestamp without timezone. + Example 3: Converting a MapType column to JSON >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |make_timestamp_ltz(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +----------------------------------------------------+ + >>> df = spark.createDataFrame([(1, {"name": "Alice"})], ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +----------------+ + |json | + +----------------+ + |{"name":"Alice"}| + +----------------+ - >>> spark.conf.unset("spark.sql.session.timeZone") + Example 4: Converting a VariantType column to JSON + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, '{"name": "Alice"}')], ("key", "value")) + >>> df.select(sf.to_json(sf.parse_json(df.value)).alias("json")).show(truncate=False) + +----------------+ + |json | + +----------------+ + |{"name":"Alice"}| + +----------------+ + + Example 5: Converting a nested MapType column to JSON + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, [{"name": "Alice"}, {"name": "Bob"}])], ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +---------------------------------+ + |json | + +---------------------------------+ + |[{"name":"Alice"},{"name":"Bob"}]| + +---------------------------------+ + + Example 6: Converting a simple ArrayType column to JSON + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, ["Alice", "Bob"])], ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +---------------+ + |json | + +---------------+ + |["Alice","Bob"]| + +---------------+ + + Example 7: Converting to JSON with specified options + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT (DATE('2022-02-22'), 1) AS date") + >>> json1 = sf.to_json(df.date) + >>> json2 = sf.to_json(df.date, {"dateFormat": "yyyy/MM/dd"}) + >>> df.select("date", json1, json2).show(truncate=False) + +---------------+------------------------------+------------------------------+ + |date |to_json(date) |to_json(date) | + +---------------+------------------------------+------------------------------+ + |{2022-02-22, 1}|{"col1":"2022-02-22","col2":1}|{"col1":"2022/02/22","col2":1}| + +---------------+------------------------------+------------------------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("to_json", _to_java_column(col), _options_to_str(options)) + + +@_try_remote_functions +def schema_of_json(json: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: + """ + Parses a JSON string and infers its schema in DDL format. + + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + json : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + a JSON string or a foldable string column containing a JSON string. + options : dict, optional + options to control parsing. accepts the same options as the JSON datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa + + .. versionchanged:: 3.0.0 + It accepts `options` parameter to control schema inferring. + + Returns + ------- + :class:`~pyspark.sql.Column` + a string representation of a :class:`StructType` parsed from given JSON. + Returns a column that evaluates to a string. + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> parsed1 = sf.schema_of_json(sf.lit('{"a": 0}')) + >>> parsed2 = sf.schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'}) + >>> spark.range(1).select(parsed1, parsed2).show() + +------------------------+----------------------+ + |schema_of_json({"a": 0})|schema_of_json({a: 1})| + +------------------------+----------------------+ + | STRUCT| STRUCT| + +------------------------+----------------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + json = _enum_to_value(json) + if not isinstance(json, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "json", + "arg_type": type(json).__name__, + }, + ) + + return _invoke_function("schema_of_json", _to_java_column(lit(json)), _options_to_str(options)) + + +@_try_remote_functions +def json_array_length(col: "ColumnOrName") -> Column: + """ + Returns the number of elements in the outermost JSON array. `NULL` is returned in case of + any other valid JSON string, `NULL` or an invalid JSON. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + col: :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + length of json array. + Returns a column that evaluates to an integer. + + Examples + -------- + >>> df = spark.createDataFrame([(None,), ('[1, 2, 3]',), ('[]',)], ['data']) + >>> df.select(json_array_length(df.data).alias('r')).collect() + [Row(r=None), Row(r=3), Row(r=0)] + """ + return _invoke_function_over_columns("json_array_length", col) + + +@_try_remote_functions +def json_object_keys(col: "ColumnOrName") -> Column: + """ + Returns all the keys of the outermost JSON object as an array. If a valid JSON object is + given, all the keys of the outermost object will be returned as an array. If it is any + other valid JSON string, an invalid JSON string or an empty string, the function returns null. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + col: :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + all the keys of the outermost JSON object. + Returns a column that evaluates to an array. + + Examples + -------- + >>> df = spark.createDataFrame([(None,), ('{}',), ('{"key1":1, "key2":2}',)], ['data']) + >>> df.select(json_object_keys(df.data).alias('r')).collect() + [Row(r=None), Row(r=[]), Row(r=['key1', 'key2'])] + """ + return _invoke_function_over_columns("json_object_keys", col) + + +@_try_remote_functions +def json_typeof(col: "ColumnOrName") -> Column: + """ + Returns the type of the outermost JSON value as a string: one of 'object', 'array', + 'string', 'number', 'boolean', or 'null'. Returns null if the input is not a valid JSON + string or is an empty string. + + .. versionadded:: 4.4.0 + + Parameters + ---------- + col: :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + the type of the outermost JSON value. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.json_object_keys` + :meth:`pyspark.sql.functions.get_json_object` + :meth:`pyspark.sql.functions.json_array_length` + + Examples + -------- + >>> df = spark.createDataFrame([('{"a": 1}',), ('[1, 2, 3]',), ('123',), ('',)], ['data']) + >>> df.select(json_typeof(df.data).alias('r')).collect() + [Row(r='object'), Row(r='array'), Row(r='number'), Row(r=None)] + """ + return _invoke_function_over_columns("json_typeof", col) + + +# ---------------------- VARIANT Functions ---------------------- + + +@_try_remote_functions +def try_parse_json( + col: "ColumnOrName", +) -> Column: + """ + Parses a column containing a JSON string into a :class:`VariantType`. Returns None if a string + contains an invalid JSON value. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column or column name JSON formatted strings. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + Returns a column that evaluates to a variant. + + Examples + -------- + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''}, {'json': '''{a : 1}'''} ]) + >>> df.select(to_json(try_parse_json(df.json))).collect() + [Row(to_json(try_parse_json(json))='{"a":1}'), Row(to_json(try_parse_json(json))=None)] + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("try_parse_json", _to_java_column(col)) + + +@_try_remote_functions +def to_variant_object( + col: "ColumnOrName", +) -> Column: + """ + Converts a column containing nested inputs (array/map/struct) into a variants where maps and + structs are converted to variant objects which are unordered unlike SQL structs. Input maps can + only have string keys. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column with a nested schema or column name + A column that evaluates to an array, map, or struct. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + Returns a column that evaluates to a variant. + + Examples + -------- + Example 1: Converting an array containing a nested struct into a variant + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, StructType, StructField, StringType, MapType + >>> schema = StructType([ + ... StructField("i", StringType(), True), + ... StructField("v", ArrayType(StructType([ + ... StructField("a", MapType(StringType(), StringType()), True) + ... ]), True)) + ... ]) + >>> data = [("1", [{"a": {"b": 2}}])] + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.to_variant_object(df.v)) + DataFrame[to_variant_object(v): variant] + >>> df.select(sf.to_variant_object(df.v)).show(truncate=False) + +--------------------+ + |to_variant_object(v)| + +--------------------+ + |[{"a":{"b":"2"}}] | + +--------------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("to_variant_object", _to_java_column(col)) + + +@_try_remote_functions +def variant_from_arrays(keys: "ColumnOrName", values: "ColumnOrName") -> Column: + """ + Creates a variant object from the given arrays of keys and values. The keys must be non-null + strings and the two arrays must have the same length. + + .. versionadded:: 4.4.0 + + Parameters + ---------- + keys : :class:`~pyspark.sql.Column` or column name + an array of string keys. + values : :class:`~pyspark.sql.Column` or column name + an array of values. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + + See Also + -------- + :meth:`pyspark.sql.functions.variant_from_entries` + :meth:`pyspark.sql.functions.to_variant_object` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT array('a', 'b') AS keys, array(1, 2) AS values") + >>> df.select(sf.variant_from_arrays("keys", "values").cast("string").alias("r")).collect() + [Row(r='{"a":1,"b":2}')] + """ + return _invoke_function_over_columns("variant_from_arrays", keys, values) + + +@_try_remote_functions +def variant_from_entries(entries: "ColumnOrName") -> Column: + """ + Creates a variant object from an array of key/value struct entries. The keys must be non-null + strings. + + .. versionadded:: 4.4.0 + + Parameters + ---------- + entries : :class:`~pyspark.sql.Column` or column name + an array of key/value structs, where the first field is a string key and the second field + is the value. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + + See Also + -------- + :meth:`pyspark.sql.functions.variant_from_arrays` + :meth:`pyspark.sql.functions.to_variant_object` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT array(struct('a', 1), struct('b', 2)) AS entries") + >>> df.select(sf.variant_from_entries("entries").cast("string").alias("r")).collect() + [Row(r='{"a":1,"b":2}')] + """ + return _invoke_function_over_columns("variant_from_entries", entries) + + +@_try_remote_functions +def parse_json( + col: "ColumnOrName", +) -> Column: + """ + Parses a column containing a JSON string into a :class:`VariantType`. Throws exception if a + string represents an invalid JSON value. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column or column name JSON formatted strings. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + Returns a column that evaluates to a variant. + + Examples + -------- + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(to_json(parse_json(df.json))).collect() + [Row(to_json(parse_json(json))='{"a":1}')] + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("parse_json", _to_java_column(col)) + + +@_try_remote_functions +def is_variant_null(v: "ColumnOrName") -> Column: + """ + Check if a variant value is a variant null. Returns true if and only if the input is a variant + null and false otherwise (including in the case of SQL NULL). + + .. versionadded:: 4.0.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + + Returns + ------- + :class:`~pyspark.sql.Column` + a boolean column indicating whether the variant value is a variant null + Returns a column that evaluates to a boolean. + + Examples + -------- + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(is_variant_null(parse_json(df.json)).alias("r")).collect() + [Row(r=False)] + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("is_variant_null", _to_java_column(v)) + + +@_try_remote_functions +def is_valid_variant(v: "ColumnOrName") -> Column: + """ + Check if a variant value is valid. Returns true if the variant is valid, false if it is + malformed, and NULL if the input is NULL. + + .. versionadded:: 4.2.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + + Returns + ------- + :class:`~pyspark.sql.Column` + a boolean column indicating whether the variant value is valid + Returns a column that evaluates to a boolean. + + Examples + -------- + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(is_valid_variant(parse_json(df.json)).alias("r")).collect() + [Row(r=True)] + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("is_valid_variant", _to_java_column(v)) + + +@_try_remote_functions +def variant_delete(v: "ColumnOrName", *paths: Union[Column, str]) -> Column: + """ + Removes fields or array elements from a variant at the given JSONPath locations. + Multiple paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are + skipped. + + .. versionadded:: 5.0.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + paths : :class:`~pyspark.sql.Column` or str + one or more JSONPath deletion targets. A `str` is a literal path; a + :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path + should start with `$` and is followed by one or more segments like + `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not + A column that evaluates to a string. + allowed. + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with the specified paths removed + Returns a column that evaluates to a variant. + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_delete + >>> df = spark.createDataFrame([{ + ... 'json': '''{ "a" : 1, "b" : 2, "c" : 3, "items" : [1, 2, 3] }''', + ... 'path': '$.a' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_delete(v, lit(None), "$.a", "$.c")).alias("r")).collect() + [Row(r='{"b":2,"items":[1,2,3]}')] + >>> df.select(to_json(variant_delete(v, "$.missing")).alias("r")).collect() + [Row(r='{"a":1,"b":2,"c":3,"items":[1,2,3]}')] + >>> df.select(to_json(variant_delete(v, df.path)).alias("r")).collect() + [Row(r='{"b":2,"c":3,"items":[1,2,3]}')] + >>> df.select(to_json(variant_delete(v, "$.items[0]", "$.items[0]")).alias("r")).collect() + [Row(r='{"a":1,"b":2,"c":3,"items":[3]}')] + >>> df.select(variant_delete(lit(None), "$.a").alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column, _to_seq + + if len(paths) == 0: + raise PySparkValueError( + errorClass="CANNOT_BE_EMPTY", + messageParameters={"item": "paths"}, + ) + sc = _get_active_spark_context() + + path_cols = [p if isinstance(p, Column) else lit(p) for p in paths] + return _invoke_function( + "variant_delete", + _to_java_column(v), + _to_java_column(path_cols[0]), + _to_seq(sc, path_cols[1:], _to_java_column), + ) + + +@_try_remote_functions +def variant_insert(v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName") -> Column: + """ + Inserts a value into a variant at the given JSONPath location. An object path adds a new field + (error if it already exists); an array path inserts at the index, shifting later elements + right. Missing intermediate keys are created. Throws an error if a path segment hits a value + of an incompatible type. Returns NULL if any argument is NULL. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath insertion target. A `str` is a literal path; a + :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with + `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or + A column that evaluates to a string. + `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to insert. Any expression castable to variant. + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with `value` inserted at `path` + Returns a column that evaluates to a variant. + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_insert + >>> df = spark.createDataFrame([{ + ... 'json': '''{ "a": 1, "arr": ["x", "y"] }''', + ... 'path': '$.d' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_insert(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"b":2}')] + >>> df.select(to_json(variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] + >>> df.select(to_json(variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","z","y"]}')] + >>> df.select(to_json(variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] + >>> df.select(to_json(variant_insert(v, df.path, lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"d":9}')] + >>> df.select(to_json(variant_insert(v, "$.b", parse_json(lit('null')))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"b":null}')] + >>> df.select(variant_insert(v, "$.b", lit(None)).alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "variant_insert", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) + + +@_try_remote_functions +def try_variant_insert( + v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" +) -> Column: + """ + Inserts a value into a variant at the given JSONPath location. An object path adds a new field; + an array path inserts at the index, shifting later elements right. Missing intermediate keys + are created. Returns NULL if the field already exists or a path segment hits a value of an + incompatible type, or if any argument is NULL. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath insertion target. A `str` is a literal path; a + :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with + `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or + `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to insert. Any expression castable to variant. + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with `value` inserted at `path`, or NULL if the insertion fails + Returns a column that evaluates to a variant. + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_insert + >>> df = spark.createDataFrame([{'json': '''{ "a": 1, "arr": ["x", "y"] }'''}]) + >>> v = parse_json(df.json) + >>> df.select(to_json(try_variant_insert(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"b":2}')] + >>> df.select(to_json(try_variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] + >>> df.select(to_json(try_variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","z","y"]}')] + >>> df.select(to_json(try_variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] + >>> df.select(to_json(try_variant_insert(v, "$.a", lit(2))).alias("r")).collect() + [Row(r=None)] + >>> df.select(to_json(try_variant_insert(v, "$.a.b", lit(2))).alias("r")).collect() + [Row(r=None)] + >>> df.select(to_json(try_variant_insert(v, "$.b", lit(None))).alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "try_variant_insert", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) + + +@_try_remote_functions +def variant_set( + v: "ColumnOrName", + path: Union[Column, str], + value: "ColumnOrName", + create_if_missing: bool = True, +) -> Column: + """ + Sets or upserts a value in a variant at the given JSONPath location. An existing object field + or array element at the target is replaced. A missing field, array index, or intermediate path + is created, unless `create_if_missing` is false, in which case the variant is left unchanged. + Throws an error if a path segment hits a value of an incompatible type. Returns NULL if any + argument is NULL. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies + the path at runtime. A valid path should start with `$` and is followed by one or more + A column that evaluates to a string. + segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to set. Any expression castable to variant. + create_if_missing : bool, optional + whether to create missing keys or out-of-range array indices (default True). + A column that evaluates to a boolean. Must be a constant. + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with `value` set at `path` + Returns a column that evaluates to a variant. + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_set + >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_set(v, "$.a", lit(9))).alias("r")).collect() + [Row(r='{"a":9,"arr":[1,2,3]}')] + >>> df.select(to_json(variant_set(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] + >>> df.select(to_json(variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,9,3]}')] + >>> df.select(to_json(variant_set(v, "$.b", lit(2), False)).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3]}')] + >>> df.select(to_json(variant_set(v, "$.a", parse_json(lit("null")))).alias("r")).collect() + [Row(r='{"a":null,"arr":[1,2,3]}')] + >>> df.select(to_json(variant_set(v, "$.a", lit(None))).alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "variant_set", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + _enum_to_value(create_if_missing), + ) + + +@_try_remote_functions +def try_variant_set( + v: "ColumnOrName", + path: Union[Column, str], + value: "ColumnOrName", + create_if_missing: bool = True, +) -> Column: + """ + Sets or upserts a value in a variant at the given JSONPath location. An existing object field + or array element at the target is replaced. A missing field, array index, or intermediate path + is created, unless `create_if_missing` is false, in which case the variant is left unchanged. + Returns NULL if a path segment hits a value of an incompatible type, or if any argument is NULL. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + path : :class:`~pyspark.sql.Column` or str + the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies + the path at runtime. A valid path should start with `$` and is followed by one or more + segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to set. Any expression castable to variant. + create_if_missing : bool, optional + whether to create missing keys or out-of-range array indices (default True). + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with `value` set at `path` + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_set + >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) + >>> v = parse_json(df.json) + >>> df.select(to_json(try_variant_set(v, "$.a", lit(9))).alias("r")).collect() + [Row(r='{"a":9,"arr":[1,2,3]}')] + >>> df.select(to_json(try_variant_set(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] + >>> df.select(to_json(try_variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,9,3]}')] + >>> df.select(to_json(try_variant_set(v, "$.arr[5]", lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3,null,null,9]}')] + >>> df.select(to_json(try_variant_set(v, "$.b", lit(2), False)).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3]}')] + >>> df.select(to_json(try_variant_set(v, "$.a.b", lit(9))).alias("r")).collect() + [Row(r=None)] + >>> df.select(to_json(try_variant_set(v, "$.a", lit(None))).alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "try_variant_set", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + _enum_to_value(create_if_missing), + ) + + +@_try_remote_functions +def variant_array_append( + v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" +) -> Column: + """ + Appends a value to the array in a variant at the given JSONPath location. Returns the variant + unchanged if a path key or index is absent. Throws an error if a path segment hits a value of + an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` + supplies the path at runtime. A valid path should start with `$` and is followed by zero or + A column that evaluates to a string. + more segments like `[123]`, `.name`, `['name']`, or `["name"]`. + value : :class:`~pyspark.sql.Column` or str + the value to append. Any expression castable to variant. + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with `value` appended to the array at `path` + Returns a column that evaluates to a variant. + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_array_append + >>> df = spark.createDataFrame([{ + ... 'json': '''[[1, 2], 5]''', + ... 'path': '$[0]' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_array_append(v, "$", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5,3]')] + >>> df.select(to_json(variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5]')] + >>> df.select(to_json(variant_array_append(v, df.path, lit(9))).alias("r")).collect() + [Row(r='[[1,2,9],5]')] + >>> nested = variant_array_append(v, "$", parse_json(lit('[4, 5]'))) + >>> df.select(to_json(nested).alias("r")).collect() + [Row(r='[[1,2],5,[4,5]]')] + >>> df.select(variant_array_append(v, "$", lit(None)).alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "variant_array_append", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) + + +@_try_remote_functions +def try_variant_array_append( + v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" +) -> Column: + """ + Appends a value to the array in a variant at the given JSONPath location. Returns the variant + unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an + incompatible type, the target is not an array, or if any argument is NULL. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + path : :class:`~pyspark.sql.Column` or str + the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` + supplies the path at runtime. A valid path should start with `$` and is followed by zero or + more segments like `[123]`, `.name`, `['name']`, or `["name"]`. + value : :class:`~pyspark.sql.Column` or str + the value to append. Any expression castable to variant. + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with `value` appended to the array at `path` + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_array_append + >>> df = spark.createDataFrame([{ + ... 'json': '''[[1, 2], 5]''', + ... 'path': '$[0]' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(try_variant_array_append(v, "$", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5,3]')] + >>> df.select(to_json(try_variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5]')] + >>> df.select(to_json(try_variant_array_append(v, df.path, lit(9))).alias("r")).collect() + [Row(r='[[1,2,9],5]')] + >>> df.select(to_json(try_variant_array_append(v, "$[1]", lit(9))).alias("r")).collect() + [Row(r=None)] + >>> df.select(try_variant_array_append(v, "$", lit(None)).alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "try_variant_array_append", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) + + +@_try_remote_functions +def variant_strip_nulls(v: "ColumnOrName", include_arrays: bool = True) -> Column: + """ + Recursively removes object fields and array elements whose value is a variant null, unless + `include_arrays` is False, in which case null array elements are kept. Returns NULL if any + argument is NULL. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + include_arrays : bool, optional + whether null elements are also removed from arrays (default True). + + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with variant null fields/elements removed + + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_strip_nulls + >>> df = spark.createDataFrame([{ + ... 'json': '''{ "a" : 1, "b" : null, "c" : [1, null], "d" : { "e" : null, "f" : 4 } }''' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_strip_nulls(v)).alias("r")).collect() + [Row(r='{"a":1,"c":[1],"d":{"f":4}}')] + >>> df.select(to_json(variant_strip_nulls(v, False)).alias("r")).collect() + [Row(r='{"a":1,"c":[1,null],"d":{"f":4}}')] + >>> df.select(variant_strip_nulls(lit(None)).alias("r")).collect() + [Row(r=None)] + >>> df2 = spark.createDataFrame([{'json': '{"a": null}'}, {'json': 'null'}]) + >>> v2 = parse_json(df2.json) + >>> df2.select(to_json(variant_strip_nulls(v2)).alias("r")).collect() + [Row(r='{}'), Row(r='null')] + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "variant_strip_nulls", _to_java_column(v), _enum_to_value(include_arrays) + ) + + +@_try_remote_functions +def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: + """ + Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to + `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + a column containing the extraction path strings or a string representing the extraction + path. A valid path should start with `$` and is followed by zero or more segments like + A column that evaluates to a string. + `[123]`, `.name`, `['name']`, or `["name"]`. + targetType : str + A DDL-formatted type string. Must be a constant. + the target data type to cast into, in a DDL-formatted string + + Returns + ------- + :class:`~pyspark.sql.Column` + a column of `targetType` representing the extracted result + Returns a column of the type given by `targetType`. + + Examples + -------- + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) + >>> df.select(variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() + [Row(r=1)] + >>> df.select(variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() + [Row(r=None)] + >>> df.select(variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() + [Row(r=1)] """ - if timezone is not None: - return _invoke_function_over_columns( - "make_timestamp_ltz", years, months, days, hours, mins, secs, timezone + from pyspark.sql.classic.column import _to_java_column + + assert isinstance(path, (Column, str)) + if isinstance(path, str): + return _invoke_function( + "variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) ) else: - return _invoke_function_over_columns( - "make_timestamp_ltz", years, months, days, hours, mins, secs + return _invoke_function( + "variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) ) @_try_remote_functions -def try_make_timestamp_ltz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: Optional["ColumnOrName"] = None, -) -> Column: +def try_variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: """ - Try to create the current timestamp with local time zone from years, months, days, hours, mins, - secs and timezone fields. - The function returns NULL on invalid inputs. + Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to + `targetType`. Returns null if the path does not exist or the cast fails. .. versionadded:: 4.0.0 Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name - The year to represent, from 1 to 9999. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name - The month-of-year to represent, from 1 (January) to 12 (December). - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name - The day-of-month to represent, from 1 to 31. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name - The hour-of-day to represent, from 0 to 23. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name - The minute-of-hour to represent, from 0 to 59. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13 , or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or column name, optional - The time zone identifier. For example, CET, UTC and etc. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + a column containing the extraction path strings or a string representing the extraction + path. A valid path should start with `$` and is followed by zero or more segments like A column that evaluates to a string. + `[123]`, `.name`, `['name']`, or `["name"]`. + targetType : str + A DDL-formatted type string. Must be a constant. + the target data type to cast into, in a DDL-formatted string Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a current timestamp, or NULL in case of an error. - Returns a column that evaluates to a timestamp. - - See Also - -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + a column of `targetType` representing the extracted result + Returns a column of the type given by `targetType`. Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - - Example 1: Make the current timestamp from years, months, days, hours, mins and secs. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec, 'tz') - ... ).show(truncate=False) - +------------------------------------------------------------+ - |try_make_timestamp_ltz(year, month, day, hour, min, sec, tz)| - +------------------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +------------------------------------------------------------+ - - Example 2: Make the current timestamp without timezone. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ltz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +--------------------------------------------------------+ - - Example 3: Make the current timestamp with invalid input. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ltz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |NULL | - +--------------------------------------------------------+ - - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) + >>> df.select(try_variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() + [Row(r=1)] + >>> df.select(try_variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() + [Row(r=None)] + >>> df.select(try_variant_get(parse_json(df.json), "$.a", "binary").alias("r")).collect() + [Row(r=None)] + >>> df.select(try_variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() + [Row(r=1)] """ - if timezone is not None: - return _invoke_function_over_columns( - "try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone + from pyspark.sql.classic.column import _to_java_column + + if isinstance(path, str): + return _invoke_function( + "try_variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) ) else: - return _invoke_function_over_columns( - "try_make_timestamp_ltz", years, months, days, hours, mins, secs + return _invoke_function( + "try_variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) ) -@overload -def make_timestamp_ntz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... - - -@overload -def make_timestamp_ntz( - *, - date: "ColumnOrName", - time: "ColumnOrName", -) -> Column: ... - - @_try_remote_functions -def make_timestamp_ntz( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, -) -> Column: +def schema_of_variant(v: "ColumnOrName") -> Column: """ - Create local date-time from years, months, days, hours, mins, secs fields. Alternatively, try to - create local date-time from date and time fields. If the configuration `spark.sql.ansi.enabled` - is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error. - - .. versionadded:: 3.5.0 + Returns schema in the SQL format of a variant. - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. + .. versionadded:: 4.0.0 Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a local date-time. - Returns a column that evaluates to a timestamp. - - See Also - -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + a string column representing the variant schema + Returns a column that evaluates to a string. Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - - Example 1: Make local date-time from years, months, days, hours, mins, secs. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |make_timestamp_ntz(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +----------------------------------------------------+ - - Example 2: Make local date-time from date and time. - - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time") - ... ) - >>> df.select(sf.make_timestamp_ntz(date=df.date, time=df.time)).show(truncate=False) - +------------------------------+ - |make_timestamp_ntz(date, time)| - +------------------------------+ - |2014-12-28 06:30:45.887 | - +------------------------------+ - - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(schema_of_variant(parse_json(df.json)).alias("r")).collect() + [Row(r='OBJECT')] """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "make_timestamp_ntz", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) - else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) - - -@overload -def try_make_timestamp_ntz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... - + from pyspark.sql.classic.column import _to_java_column -@overload -def try_make_timestamp_ntz( - *, - date: "ColumnOrName", - time: "ColumnOrName", -) -> Column: ... + return _invoke_function("schema_of_variant", _to_java_column(v)) @_try_remote_functions -def try_make_timestamp_ntz( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, -) -> Column: +def schema_of_variant_agg(v: "ColumnOrName") -> Column: """ - Try to create local date-time from years, months, days, hours, mins, secs fields. Alternatively, - try to create local date-time from date and time fields. The function returns NULL on invalid - inputs. + Returns the merged schema in the SQL format of a variant column. .. versionadded:: 4.0.0 - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. - Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a local date-time, or NULL in case of an error. - Returns a column that evaluates to a timestamp. + a string column representing the variant schema + Returns a column that evaluates to a string. - See Also + Examples -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(schema_of_variant_agg(parse_json(df.json)).alias("r")).collect() + [Row(r='OBJECT')] + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("schema_of_variant_agg", _to_java_column(v)) + + +# ---------------------- XML Functions ---------------------- + + +@_try_remote_functions +def xpath(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a string array of values within the nodes of xml that match the XPath expression. + + .. versionadded:: 3.5.0 Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [('b1b2b3c1c2',)], ['x']) + >>> df.select(sf.xpath(df.x, sf.lit('a/b/text()'))).show() + +--------------------+ + |xpath(x, a/b/text())| + +--------------------+ + | [b1, b2, b3]| + +--------------------+ - Example 1: Make local date-time from years, months, days, hours, mins, secs. + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath", xml, path) - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ntz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +--------------------------------------------------------+ - Example 2: Make local date-time with invalid input +@_try_remote_functions +def xpath_boolean(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns true if the XPath expression evaluates to true, or if a matching node is found. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ntz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |NULL | - +--------------------------------------------------------+ + .. versionadded:: 3.5.0 - >>> spark.conf.unset("spark.sql.session.timeZone") - """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "try_make_timestamp_ntz", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) - else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "try_make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('1',)], ['x']) + >>> df.select(sf.xpath_boolean(df.x, sf.lit('a/b'))).show() + +---------------------+ + |xpath_boolean(x, a/b)| + +---------------------+ + | true| + +---------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_boolean", xml, path) @_try_remote_functions -def make_ym_interval( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, -) -> Column: +def xpath_double(xml: "ColumnOrName", path: "ColumnOrName") -> Column: """ - Make year-month interval from years, months. + Returns a double value, the value zero if no match is found, + or NaN if a match is found but the value is non-numeric. .. versionadded:: 3.5.0 - Parameters - ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative. - A column that evaluates to an integer. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains a year-month interval. - Returns a column that evaluates to an interval. - - See Also - -------- - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.make_dt_interval` - :meth:`pyspark.sql.functions.try_make_interval` - Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - - Example 1: Make year-month interval from years, months. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_double(df.x, sf.lit('sum(a/b)'))).show() + +-------------------------+ + |xpath_double(x, sum(a/b))| + +-------------------------+ + | 3.0| + +-------------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) - >>> df.select('*', sf.make_ym_interval('year', df.month)).show(truncate=False) - +----+-----+-------------------------------+ - |year|month|make_ym_interval(year, month) | - +----+-----+-------------------------------+ - |2014|12 |INTERVAL '2015-0' YEAR TO MONTH| - +----+-----+-------------------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_double", xml, path) - Example 2: Make year-month interval from years. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) - >>> df.select('*', sf.make_ym_interval(df.year)).show(truncate=False) - +----+-----+-------------------------------+ - |year|month|make_ym_interval(year, 0) | - +----+-----+-------------------------------+ - |2014|12 |INTERVAL '2014-0' YEAR TO MONTH| - +----+-----+-------------------------------+ +@_try_remote_functions +def xpath_number(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a double value, the value zero if no match is found, + or NaN if a match is found but the value is non-numeric. - Example 3: Make empty interval. + .. versionadded:: 3.5.0 + Examples + -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_ym_interval()).show(truncate=False) - +----------------------------+ - |make_ym_interval(0, 0) | - +----------------------------+ - |INTERVAL '0-0' YEAR TO MONTH| - +----------------------------+ + >>> spark.createDataFrame( + ... [('12',)], ['x'] + ... ).select(sf.xpath_number('x', sf.lit('sum(a/b)'))).show() + +-------------------------+ + |xpath_number(x, sum(a/b))| + +-------------------------+ + | 3.0| + +-------------------------+ - >>> spark.conf.unset("spark.sql.session.timeZone") + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` """ - _years = lit(0) if years is None else years - _months = lit(0) if months is None else months - return _invoke_function_over_columns("make_ym_interval", _years, _months) + return _invoke_function_over_columns("xpath_number", xml, path) @_try_remote_functions -def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column: +def xpath_float(xml: "ColumnOrName", path: "ColumnOrName") -> Column: """ - Partition transform function: A transform for any type that partitions - by a hash of the input column. - - .. versionadded:: 3.1.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Returns a float value, the value zero if no match is found, + or NaN if a match is found but the value is non-numeric. - .. deprecated:: 4.0.0 - Use :func:`partitioning.bucket` instead. + .. versionadded:: 3.5.0 Examples -------- - >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP - ... bucket(42, "ts") - ... ).createOrReplace() - - Parameters - ---------- - numBuckets : :class:`~pyspark.sql.Column` or int - the number of buckets - col : :class:`~pyspark.sql.Column` or str - target date or timestamp column to work on. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_float(df.x, sf.lit('sum(a/b)'))).show() + +------------------------+ + |xpath_float(x, sum(a/b))| + +------------------------+ + | 3.0| + +------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - data partitioned by given columns. + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_float", xml, path) - Notes - ----- - This function can be used only in combination with - :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. +@_try_remote_functions +def xpath_int(xml: "ColumnOrName", path: "ColumnOrName") -> Column: """ - from pyspark.sql.functions import partitioning - - warnings.warn("Deprecated in 4.0.0, use partitioning.bucket instead.", FutureWarning) + Returns an integer value, or the value zero if no match is found, + or a match is found but the value is non-numeric. - return partitioning.bucket(numBuckets, col) + .. versionadded:: 3.5.0 + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_int(df.x, sf.lit('sum(a/b)'))).show() + +----------------------+ + |xpath_int(x, sum(a/b))| + +----------------------+ + | 3| + +----------------------+ -# Geospatial ST Functions + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_int", xml, path) @_try_remote_functions -def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None) -> Column: - """Returns the input GEOGRAPHY or GEOMETRY value in WKB format. - - .. versionadded:: 4.1.0 - - .. versionchanged:: 4.2.0 - Added the optional `endianness` parameter. +def xpath_long(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a long integer value, or the value zero if no match is found, + or a match is found but the value is non-numeric. - Parameters - ---------- - geo : :class:`~pyspark.sql.Column` or str - A geospatial value, either a GEOGRAPHY or a GEOMETRY. - endianness : :class:`~pyspark.sql.Column` or str, optional - The optional endianness of the output WKB, 'NDR' for little-endian (default) or 'XDR' for - big-endian. + .. versionadded:: 3.5.0 Examples -------- - - Example 1: Getting WKB from GEOGRAPHY. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_long(df.x, sf.lit('sum(a/b)'))).show() + +-----------------------+ + |xpath_long(x, sum(a/b))| + +-----------------------+ + | 3| + +-----------------------+ - Example 2: Getting WKB from GEOMETRY. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_long", xml, path) - Example 3: Getting WKB (little-endian) from GEOGRAPHY. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb'), 'NDR'))).collect() - [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] - Example 4: Getting WKB (big-endian) from GEOMETRY. +@_try_remote_functions +def xpath_short(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a short integer value, or the value zero if no match is found, + or a match is found but the value is non-numeric. + + .. versionadded:: 3.5.0 + + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb'), 'XDR'))).collect() - [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), XDR))='00000000013FF00000000000004000000000000000')] + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_short(df.x, sf.lit('sum(a/b)'))).show() + +------------------------+ + |xpath_short(x, sum(a/b))| + +------------------------+ + | 3| + +------------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_string` """ - if endianness is None: - return _invoke_function_over_columns("st_asbinary", geo) - else: - _endianness = lit(endianness) if isinstance(endianness, str) else endianness - return _invoke_function_over_columns("st_asbinary", geo, _endianness) + return _invoke_function_over_columns("xpath_short", xml, path) @_try_remote_functions -def st_geogfromwkb(wkb: "ColumnOrName") -> Column: - """Parses the input WKB description and returns the corresponding GEOGRAPHY value. - - .. versionadded:: 4.1.0 +def xpath_string(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns the text contents of the first xml node that matches the XPath expression. - Parameters - ---------- - wkb : :class:`~pyspark.sql.Column` or str - A BINARY value in WKB format, representing a GEOGRAPHY value. - A column that evaluates to a binary. + .. versionadded:: 3.5.0 Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] + >>> df = spark.createDataFrame([('bcc',)], ['x']) + >>> df.select(sf.xpath_string(df.x, sf.lit('a/c'))).show() + +--------------------+ + |xpath_string(x, a/c)| + +--------------------+ + | cc| + +--------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` """ - return _invoke_function_over_columns("st_geogfromwkb", wkb) + return _invoke_function_over_columns("xpath_string", xml, path) +# TODO: Fix and add an example for StructType with Spark Connect +# e.g., StructType([StructField("a", IntegerType())]) @_try_remote_functions -def st_geomfromwkb( - wkb: "ColumnOrName", srid: Optional[Union["ColumnOrName", int]] = None +def from_xml( + col: "ColumnOrName", + schema: Union[StructType, Column, str], + options: Optional[Mapping[str, str]] = None, ) -> Column: - """Parses the input WKB description and returns the corresponding GEOMETRY value. + """ + Parses a column containing a XML string to a row with + the specified schema. Returns `null`, in the case of an unparsable string. - .. versionadded:: 4.1.0 + .. versionadded:: 4.0.0 Parameters ---------- - wkb : :class:`~pyspark.sql.Column` or str - A BINARY value in WKB format, representing a GEOMETRY value. - A column that evaluates to a binary. - srid : :class:`~pyspark.sql.Column` or int, optional - The optional SRID value of the geometry. Default is 0. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + a column or column name in XML format + schema : :class:`StructType`, :class:`~pyspark.sql.Column` or str + a StructType, Column or Python string literal with a DDL-formatted string + A column that evaluates to a string, or a DDL-formatted type string, or a DataType. + to use when parsing the Xml column + options : dict, optional + options to control parsing. accepts the same options as the Xml datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa + + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of complex type from given XML object. + Returns a column that evaluates to a struct. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] + Example 1: Parsing XML with a DDL-formatted string schema + + >>> import pyspark.sql.functions as sf + >>> data = [(1, '''

1

''')] + >>> df = spark.createDataFrame(data, ("key", "value")) + ... # Define the schema using a DDL-formatted string + >>> schema = "STRUCT" + ... # Parse the XML column using the DDL-formatted schema + >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() + [Row(xml=Row(a=1))] + + Example 2: Parsing XML with a :class:`StructType` schema + + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql.types import StructType, LongType + >>> data = [(1, '''

1

''')] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> schema = StructType().add("a", LongType()) + >>> df.select(sf.from_xml(df.value, schema)).show() + +---------------+ + |from_xml(value)| + +---------------+ + | {1}| + +---------------+ + + Example 3: Parsing XML with :class:`ArrayType` in schema + + >>> import pyspark.sql.functions as sf + >>> data = [(1, '

12

')] + >>> df = spark.createDataFrame(data, ("key", "value")) + ... # Define the schema with an Array type + >>> schema = "STRUCT>" + ... # Parse the XML column using the schema with an Array + >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() + [Row(xml=Row(a=[1, 2]))] + + Example 4: Parsing XML using :meth:`pyspark.sql.functions.schema_of_xml` + + >>> import pyspark.sql.functions as sf + >>> # Sample data with an XML column + ... data = [(1, '

12

')] + >>> df = spark.createDataFrame(data, ("key", "value")) + ... # Generate the schema from an example XML value + >>> schema = sf.schema_of_xml(sf.lit(data[0][1])) + ... # Parse the XML column using the generated schema + >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() + [Row(xml=Row(a=[1, 2]))] + + See Also + -------- + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` """ - if srid is None: - return _invoke_function_over_columns("st_geomfromwkb", wkb) - else: - srid = _enum_to_value(srid) - srid = lit(srid) if isinstance(srid, int) else srid - return _invoke_function_over_columns("st_geomfromwkb", wkb, srid) + from pyspark.sql.classic.column import _to_java_column + + if isinstance(schema, StructType): + schema = schema.json() + elif isinstance(schema, Column): + schema = _to_java_column(schema) + elif not isinstance(schema, str): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "StructType, Column or str", + "arg_name": "schema", + "arg_type": type(schema).__name__, + }, + ) + return _invoke_function("from_xml", _to_java_column(col), schema, _options_to_str(options)) @_try_remote_functions -def st_setsrid(geo: "ColumnOrName", srid: Union["ColumnOrName", int]) -> Column: - """Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. +def schema_of_xml(xml: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: + """ + Parses a XML string and infers its schema in DDL format. - .. versionadded:: 4.1.0 + .. versionadded:: 4.0.0 Parameters ---------- - geo : :class:`~pyspark.sql.Column` or str - A geospatial value, either a GEOGRAPHY or a GEOMETRY. - srid : :class:`~pyspark.sql.Column` or int - An INTEGER representing the new SRID of the geospatial value. + xml : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + a XML string or a foldable string column containing a XML string. + options : dict, optional + options to control parsing. accepts the same options as the XML datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa + + Returns + ------- + :class:`~pyspark.sql.Column` + a string representation of a :class:`StructType` parsed from given XML. + Returns a column that evaluates to a string. Examples -------- + Example 1: Parsing a simple XML with a single element - Example 1: Setting the SRID on GEOGRAPHY with SRID from another column. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'), 4326)], ['wkb', 'srid']) # noqa - >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geogfromwkb('wkb'), 'srid'))).collect() - [Row(st_srid(st_setsrid(st_geogfromwkb(wkb), srid))=4326)] + >>> df = spark.range(1) + >>> df.select(sf.schema_of_xml(sf.lit('

1

')).alias("xml")).collect() + [Row(xml='STRUCT')] + + Example 2: Parsing an XML with multiple elements in an array - Example 2: Setting the SRID on GEOMETRY with SRID as an integer literal. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geomfromwkb('wkb'), 4326))).collect() - [Row(st_srid(st_setsrid(st_geomfromwkb(wkb, 0), 4326))=4326)] - """ - srid = _enum_to_value(srid) - srid = lit(srid) if isinstance(srid, int) else srid - return _invoke_function_over_columns("st_setsrid", geo, srid) + >>> df.select(sf.schema_of_xml(sf.lit('

12

')).alias("xml")).collect() + [Row(xml='STRUCT>')] + Example 3: Parsing XML with options to exclude attributes -@_try_remote_functions -def st_srid(geo: "ColumnOrName") -> Column: - """Returns the SRID of the input GEOGRAPHY or GEOMETRY value. + >>> from pyspark.sql import functions as sf + >>> schema = sf.schema_of_xml('

1

', {'excludeAttribute':'true'}) + >>> df.select(schema.alias("xml")).collect() + [Row(xml='STRUCT')] - .. versionadded:: 4.1.0 + Example 4: Parsing XML with complex structure - Parameters - ---------- - geo : :class:`~pyspark.sql.Column` or str - A geospatial value, either a GEOGRAPHY or a GEOMETRY. + >>> from pyspark.sql import functions as sf + >>> df.select( + ... sf.schema_of_xml( + ... sf.lit('Alice30') + ... ).alias("xml") + ... ).collect() + [Row(xml='STRUCT>')] - Examples - -------- + Example 5: Parsing XML with nested arrays - Example 1: Getting the SRID of GEOGRAPHY. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.st_srid(sf.st_geogfromwkb('wkb'))).collect() - [Row(st_srid(st_geogfromwkb(wkb))=4326)] + >>> df.select( + ... sf.schema_of_xml( + ... sf.lit('12') + ... ).alias("xml") + ... ).collect() + [Row(xml='STRUCT>>')] - Example 2: Getting the SRID of GEOMETRY. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.st_srid(sf.st_geomfromwkb('wkb'))).collect() - [Row(st_srid(st_geomfromwkb(wkb, 0))=0)] + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` """ - return _invoke_function_over_columns("st_srid", geo) + from pyspark.sql.classic.column import _to_java_column + xml = _enum_to_value(xml) + if not isinstance(xml, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "xml", + "arg_type": type(xml).__name__, + }, + ) -# Call Functions + return _invoke_function("schema_of_xml", _to_java_column(lit(xml)), _options_to_str(options)) @_try_remote_functions -def call_udf(udfName: str, *cols: "ColumnOrName") -> Column: +def to_xml(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: """ - Call a user-defined function. + Converts a column containing a :class:`StructType` into a XML string. + Throws an exception, in the case of an unsupported type. - .. versionadded:: 3.4.0 + .. versionadded:: 4.0.0 Parameters ---------- - udfName : str - name of the user defined function (UDF) - cols : :class:`~pyspark.sql.Column` or str - column names or :class:`~pyspark.sql.Column`\\s to be used in the UDF + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a struct, array, map, or variant. + name of column containing a struct. + options: dict, optional + options to control converting. accepts the same options as the XML datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - result of executed udf. + a XML string converted from given :class:`StructType`. + Returns a column that evaluates to a string. Examples -------- - >>> from pyspark.sql.functions import call_udf, col - >>> from pyspark.sql.types import IntegerType, StringType - >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) - >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) - >>> df.select(call_udf("intX2", "id")).show() - +---------+ - |intX2(id)| - +---------+ - | 2| - | 4| - | 6| - +---------+ - >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) - >>> df.select(call_udf("strX2", col("name"))).show() - +-----------+ - |strX2(name)| - +-----------+ - | aa| - | bb| - | cc| - +-----------+ + >>> from pyspark.sql import Row + >>> data = [(1, Row(age=2, name='Alice'))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(to_xml(df.value, {'rowTag':'person'}).alias("xml")).collect() + [Row(xml='\\n 2\\n Alice\\n')] + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + from pyspark.sql.classic.column import _to_java_column - sc = _get_active_spark_context() - return _invoke_function("call_udf", udfName, _to_seq(sc, cols, _to_java_column)) + return _invoke_function("to_xml", _to_java_column(col), _options_to_str(options)) + + +# ---------------------- URL Functions ---------------------- @_try_remote_functions -def call_function(funcName: str, *cols: "ColumnOrName") -> Column: +def try_parse_url( + url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None +) -> Column: """ - Call a SQL function. + This is a special version of `parse_url` that performs the same operation, but returns a + NULL value instead of raising an error if the parsing cannot be performed. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - funcName : str - function name that follows the SQL identifier syntax (can be quoted, can be qualified) - cols : :class:`~pyspark.sql.Column` or str - column names or :class:`~pyspark.sql.Column`\\s to be used in the function + url : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL. + A column that evaluates to a string. + partToExtract : :class:`~pyspark.sql.Column` or str + A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. + key : :class:`~pyspark.sql.Column` or str, optional + A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - result of executed function. + A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- - >>> from pyspark.sql.functions import call_udf, col - >>> from pyspark.sql.types import IntegerType, StringType - >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) - >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) - >>> df.select(call_function("intX2", "id")).show() - +---------+ - |intX2(id)| - +---------+ - | 2| - | 4| - | 6| - +---------+ - >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) - >>> df.select(call_function("strX2", col("name"))).show() - +-----------+ - |strX2(name)| - +-----------+ - | aa| - | bb| - | cc| - +-----------+ - >>> df.select(call_function("avg", col("id"))).show() - +-------+ - |avg(id)| - +-------+ - | 2.0| - +-------+ - >>> _ = spark.sql("CREATE FUNCTION custom_avg AS 'test.org.apache.spark.sql.MyDoubleAvg'") - ... # doctest: +SKIP - >>> df.select(call_function("custom_avg", col("id"))).show() - ... # doctest: +SKIP - +------------------------------------+ - |spark_catalog.default.custom_avg(id)| - +------------------------------------+ - | 102.0| - +------------------------------------+ - >>> df.select(call_function("spark_catalog.default.custom_avg", col("id"))).show() - ... # doctest: +SKIP - +------------------------------------+ - |spark_catalog.default.custom_avg(id)| - +------------------------------------+ - | 102.0| - +------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + Example 1: Extracting the query part from a URL - sc = _get_active_spark_context() - return _invoke_function("call_function", funcName, _to_seq(sc, cols, _to_java_column)) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "QUERY")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() + +------------------------+ + |try_parse_url(url, part)| + +------------------------+ + | query=1| + +------------------------+ + Example 2: Extracting the value of a specific query parameter from a URL -@_try_remote_functions -def unwrap_udt(col: "ColumnOrName") -> Column: - """ - Unwrap UDT data type column into its underlying type. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], + ... ["url", "part", "key"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() + +-----------------------------+ + |try_parse_url(url, part, key)| + +-----------------------------+ + | 1| + +-----------------------------+ - .. versionadded:: 3.4.0 + Example 3: Extracting the protocol part from a URL - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() + +------------------------+ + |try_parse_url(url, part)| + +------------------------+ + | https| + +------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - The underlying representation. + Example 4: Extracting the host part from a URL - See Also - -------- - :meth:`pyspark.sql.functions.wrap_udt` + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "HOST")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() + +------------------------+ + |try_parse_url(url, part)| + +------------------------+ + | spark.apache.org| + +------------------------+ - Examples - -------- - Example 1: Unwrap ML-specific UDT - VectorUDT + Example 5: Extracting the path part from a URL >>> from pyspark.sql import functions as sf - >>> from pyspark.ml.linalg import Vectors - >>> vec1 = Vectors.dense(1, 2, 3) - >>> vec2 = Vectors.sparse(4, {1: 1.0, 3: 5.5}) - >>> df = spark.createDataFrame([(vec1,), (vec2,)], ["vec"]) - >>> df.select(sf.unwrap_udt("vec")).printSchema() - root - |-- unwrap_udt(vec): struct (nullable = true) - | |-- type: byte (nullable = false) - | |-- size: integer (nullable = true) - | |-- indices: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- values: array (nullable = true) - | | |-- element: double (containsNull = false) + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PATH")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() + +------------------------+ + |try_parse_url(url, part)| + +------------------------+ + | /path| + +------------------------+ - Example 2: Unwrap ML-specific UDT - MatrixUDT + Example 6: Invalid URL >>> from pyspark.sql import functions as sf - >>> from pyspark.ml.linalg import Matrices - >>> mat1 = Matrices.dense(2, 2, range(4)) - >>> mat2 = Matrices.sparse(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4]) - >>> df = spark.createDataFrame([(mat1,), (mat2,)], ["mat"]) - >>> df.select(sf.unwrap_udt("mat")).printSchema() - root - |-- unwrap_udt(mat): struct (nullable = true) - | |-- type: byte (nullable = false) - | |-- numRows: integer (nullable = false) - | |-- numCols: integer (nullable = false) - | |-- colPtrs: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- rowIndices: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- values: array (nullable = true) - | | |-- element: double (containsNull = false) - | |-- isTransposed: boolean (nullable = false) + >>> df = spark.createDataFrame( + ... [("inva lid://spark.apache.org/path?query=1", "QUERY", "query")], + ... ["url", "part", "key"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() + +-----------------------------+ + |try_parse_url(url, part, key)| + +-----------------------------+ + | NULL| + +-----------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("unwrap_udt", _to_java_column(col)) + if key is not None: + return _invoke_function_over_columns("try_parse_url", url, partToExtract, key) + else: + return _invoke_function_over_columns("try_parse_url", url, partToExtract) @_try_remote_functions -def wrap_udt(col: "ColumnOrName", udt: "Union[UserDefinedType, Column]") -> Column: +def parse_url( + url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None +) -> Column: """ - Wrap a column as a user-defined type. + URL function: Extracts a specified part from a URL. If a key is provided, + it returns the associated query parameter value. - .. versionadded:: 4.4.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column to wrap. The column data type must match the UDT's underlying SQL type. - udt : :class:`~pyspark.sql.types.UserDefinedType` or :class:`~pyspark.sql.Column` - The target user-defined type, or a constant string column containing its JSON - representation. + url : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL. + A column that evaluates to a string. + partToExtract : :class:`~pyspark.sql.Column` or str + A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. + key : :class:`~pyspark.sql.Column` or str, optional + A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A column of the target user-defined type. - - See Also - -------- - :meth:`pyspark.sql.functions.unwrap_udt` + A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- - Example 1: Wrapping a vector struct as VectorUDT + Example 1: Extracting the query part from a URL >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Row - >>> from pyspark.sql.types import StructField, StructType - >>> from pyspark.ml.linalg import VectorUDT - >>> vector_schema = StructType([StructField("vec", VectorUDT.sqlType(), True)]) >>> df = spark.createDataFrame( - ... [(Row(type=1, size=None, indices=None, values=[1.0, 2.0, 3.0]),)], - ... vector_schema) - >>> df.select("*", sf.wrap_udt("vec", VectorUDT())).show() - +--------------------+...+ - | vec|wrap_udt(vec...| - +--------------------+...+ - |{1, NULL, NULL, [...|...[1.0,2.0,3.0]| - +--------------------+...+ - >>> row = df.select(sf.wrap_udt("vec", VectorUDT())).first() - >>> type(row[0]) - + ... [("https://spark.apache.org/path?query=1", "QUERY")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | query=1| + +--------------------+ - Example 2: Wrapping a matrix struct as MatrixUDT + Example 2: Extracting the value of a specific query parameter from a URL >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Row - >>> from pyspark.sql.types import StructField, StructType - >>> from pyspark.mllib.linalg import MatrixUDT - >>> matrix_schema = StructType([StructField("mat", MatrixUDT.sqlType(), True)]) >>> df = spark.createDataFrame( - ... [( - ... Row( - ... type=1, - ... numRows=2, - ... numCols=2, - ... colPtrs=None, - ... rowIndices=None, - ... values=[1.0, 2.0, 3.0, 4.0], - ... isTransposed=False), - ... )], - ... matrix_schema) - >>> df.select("*", sf.wrap_udt("mat", MatrixUDT())).printSchema() - root - |-- mat: struct (nullable = true) - | |-- type: byte (nullable = false) - | |-- numRows: integer (nullable = false) - | |-- numCols: integer (nullable = false) - | |-- colPtrs: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- rowIndices: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- values: array (nullable = true) - | | |-- element: double (containsNull = false) - | |-- isTransposed: boolean (nullable = false) - |-- wrap_udt(mat...: matrix... (nullable = true) - >>> row = df.select(sf.wrap_udt("mat", MatrixUDT())).first() - >>> type(row[0]) - - """ - from pyspark.sql.classic.column import _to_java_column + ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], + ... ["url", "part", "key"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part, df.key)).show() + +-------------------------+ + |parse_url(url, part, key)| + +-------------------------+ + | 1| + +-------------------------+ - if isinstance(udt, _UserDefinedType): - udt_col = lit(udt.json()) - elif isinstance(udt, Column): - udt_col = udt - else: - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "UserDefinedType or Column", - "arg_name": "udt", - "arg_type": type(udt).__name__, - }, - ) - return _invoke_function("wrap_udt", _to_java_column(col), _to_java_column(udt_col)) + Example 3: Extracting the protocol part from a URL + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | https| + +--------------------+ + + Example 4: Extracting the host part from a URL + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "HOST")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | spark.apache.org| + +--------------------+ + Example 5: Extracting the path part from a URL -# ---------------------- Datasketch functions ------------------------------ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PATH")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | /path| + +--------------------+ + """ + if key is not None: + return _invoke_function_over_columns("parse_url", url, partToExtract, key) + else: + return _invoke_function_over_columns("parse_url", url, partToExtract) @_try_remote_functions -def hll_sketch_agg( - col: "ColumnOrName", - lgConfigK: Optional[Union[int, Column]] = None, -) -> Column: +def url_decode(str: "ColumnOrName") -> Column: """ - Aggregate function: returns the updatable binary representation of the Datasketches - HllSketch configured with lgConfigK arg. + URL function: Decodes a URL-encoded string in 'application/x-www-form-urlencoded' + format to its original format. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to an integer, long, string, or binary. - lgConfigK : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of K, where K is the number of buckets or slots for the HllSketch. - A column that evaluates to an integer. + str : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the HllSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.hll_union` - :meth:`pyspark.sql.functions.hll_union_agg` - :meth:`pyspark.sql.functions.hll_sketch_estimate` + A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- + Example 1: Decoding a URL-encoded string + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() - +----------------------------------------------+ - |hll_sketch_estimate(hll_sketch_agg(value, 12))| - +----------------------------------------------+ - | 3| - +----------------------------------------------+ + >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show(truncate=False) + +------------------------+ + |url_decode(url) | + +------------------------+ + |https://spark.apache.org| + +------------------------+ - >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value", 12))).show() - +----------------------------------------------+ - |hll_sketch_estimate(hll_sketch_agg(value, 12))| - +----------------------------------------------+ - | 3| - +----------------------------------------------+ - """ - if lgConfigK is None: - return _invoke_function_over_columns("hll_sketch_agg", col) - else: - return _invoke_function_over_columns("hll_sketch_agg", col, lit(lgConfigK)) + Example 2: Decoding a URL-encoded string with spaces + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Hello%20World%21",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | Hello World!| + +---------------+ -@_try_remote_functions -def hll_union_agg( - col: "ColumnOrName", - allowDifferentLgConfigK: Optional[Union[bool, Column]] = None, -) -> Column: - """ - Aggregate function: returns the updatable binary representation of the Datasketches - HllSketch, generated by merging previously created Datasketches HllSketch instances - via a Datasketches Union instance. Throws an exception if sketches have different - lgConfigK values and allowDifferentLgConfigK is unset or set to false. + Example 3: Decoding a URL-encoded string with special characters - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("A%2BB%3D%3D",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | A+B==| + +---------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - allowDifferentLgConfigK : :class:`~pyspark.sql.Column` or bool, optional - Allow sketches with different lgConfigK values to be merged (defaults to false). + Example 4: Decoding a URL-encoded string with non-ASCII characters - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the merged HllSketch. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("%E4%BD%A0%E5%A5%BD",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | 你好| + +---------------+ - See Also - -------- - :meth:`pyspark.sql.functions.hll_union` - :meth:`pyspark.sql.functions.hll_sketch_agg` - :meth:`pyspark.sql.functions.hll_sketch_estimate` + Example 5: Decoding a URL-encoded string with hexadecimal values - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,2,3], "INT") - >>> df1 = df1.agg(sf.hll_sketch_agg("value").alias("sketch")) - >>> df2 = spark.createDataFrame([4,5,5,6], "INT") - >>> df2 = df2.agg(sf.hll_sketch_agg("value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch"))).show() - +-------------------------------------------------+ - |hll_sketch_estimate(hll_union_agg(sketch, false))| - +-------------------------------------------------+ - | 6| - +-------------------------------------------------+ - - >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch", False))).show() - +-------------------------------------------------+ - |hll_sketch_estimate(hll_union_agg(sketch, false))| - +-------------------------------------------------+ - | 6| - +-------------------------------------------------+ + >>> df = spark.createDataFrame([("%7E%21%40%23%24%25%5E%26%2A%28%29%5F%2B",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | ~!@#$%^&*()_+| + +---------------+ """ - if allowDifferentLgConfigK is None: - return _invoke_function_over_columns("hll_union_agg", col) - else: - return _invoke_function_over_columns("hll_union_agg", col, lit(allowDifferentLgConfigK)) + return _invoke_function_over_columns("url_decode", str) @_try_remote_functions -def hll_sketch_estimate(col: "ColumnOrName") -> Column: +def try_url_decode(str: "ColumnOrName") -> Column: """ - Returns the estimated number of unique values given the binary representation - of a Datasketches HllSketch. + This is a special version of `url_decode` that performs the same operation, but returns a + NULL value instead of raising an error if the decoding cannot be performed. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + str : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - The estimated number of unique values for the HllSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.hll_union` - :meth:`pyspark.sql.functions.hll_union_agg` - :meth:`pyspark.sql.functions.hll_sketch_agg` + A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- + Example 1: Decoding a URL-encoded string + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() - +----------------------------------------------+ - |hll_sketch_estimate(hll_sketch_agg(value, 12))| - +----------------------------------------------+ - | 3| - +----------------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) + >>> df.select(sf.try_url_decode(df.url)).show(truncate=False) + +------------------------+ + |try_url_decode(url) | + +------------------------+ + |https://spark.apache.org| + +------------------------+ - return _invoke_function("hll_sketch_estimate", _to_java_column(col)) + Example 2: Return NULL if the decoding cannot be performed. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("https%3A%2F%2spark.apache.org",)], ["url"]) + >>> df.select(sf.try_url_decode(df.url)).show() + +-------------------+ + |try_url_decode(url)| + +-------------------+ + | NULL| + +-------------------+ + """ + return _invoke_function_over_columns("try_url_decode", str) @_try_remote_functions -def hll_union( - col1: "ColumnOrName", col2: "ColumnOrName", allowDifferentLgConfigK: Optional[bool] = None -) -> Column: +def url_encode(str: "ColumnOrName") -> Column: """ - Merges two binary representations of Datasketches HllSketch objects, using a - Datasketches Union object. Throws an exception if sketches have different - lgConfigK values and allowDifferentLgConfigK is unset or set to false. + URL function: Encodes a string into a URL-encoded string in + 'application/x-www-form-urlencoded' format. .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - col2 : :class:`~pyspark.sql.Column` or column name - allowDifferentLgConfigK : bool, optional - Allow sketches with different lgConfigK values to be merged (defaults to false). + str : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a string to be URL-encoded. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged HllSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.hll_union_agg` - :meth:`pyspark.sql.functions.hll_sketch_agg` - :meth:`pyspark.sql.functions.hll_sketch_estimate` + A new column of strings, each representing the URL-encoded string. + Returns a column that evaluates to a string. Examples -------- + Example 1: Encoding a simple URL + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,4),(2,5),(2,5),(3,6)], "struct") - >>> df = df.agg( - ... sf.hll_sketch_agg("v1").alias("sketch1"), - ... sf.hll_sketch_agg("v2").alias("sketch2") - ... ) - >>> df.select(sf.hll_sketch_estimate(sf.hll_union(df.sketch1, "sketch2"))).show() - +-------------------------------------------------------+ - |hll_sketch_estimate(hll_union(sketch1, sketch2, false))| - +-------------------------------------------------------+ - | 6| - +-------------------------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + >>> df = spark.createDataFrame([("https://spark.apache.org",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show(truncate=False) + +------------------------------+ + |url_encode(url) | + +------------------------------+ + |https%3A%2F%2Fspark.apache.org| + +------------------------------+ - if allowDifferentLgConfigK is not None: - return _invoke_function( - "hll_union", - _to_java_column(col1), - _to_java_column(col2), - _enum_to_value(allowDifferentLgConfigK), - ) - else: - return _invoke_function("hll_union", _to_java_column(col1), _to_java_column(col2)) + Example 2: Encoding a URL with spaces + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Hello World!",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show() + +---------------+ + |url_encode(url)| + +---------------+ + | Hello+World%21| + +---------------+ -@_try_remote_functions -def theta_sketch_agg( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - ThetaSketch with the values in the input column configured with lgNomEntries nominal entries. + Example 3: Encoding a URL with special characters - .. versionadded:: 4.1.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("A+B==",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show() + +---------------+ + |url_encode(url)| + +---------------+ + | A%2BB%3D%3D| + +---------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to an array, binary, double, float, integer, long, or string. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries, where nominal entries is the size of the sketch - (must be between 4 and 26, defaults to 12). - A column that evaluates to an integer. + Example 4: Encoding a URL with non-ASCII characters - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the ThetaSketch. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("你好",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show() + +------------------+ + | url_encode(url)| + +------------------+ + |%E4%BD%A0%E5%A5%BD| + +------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_intersection` - :meth:`pyspark.sql.functions.theta_difference` - :meth:`pyspark.sql.functions.theta_union_agg` - :meth:`pyspark.sql.functions.theta_intersection_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + Example 5: Encoding a URL with hexadecimal values - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_sketch_agg(value, 12))| - +--------------------------------------------------+ - | 3| - +--------------------------------------------------+ - - >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value", 15))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_sketch_agg(value, 15))| - +--------------------------------------------------+ - | 3| - +--------------------------------------------------+ + >>> df = spark.createDataFrame([("~!@#$%^&*()_+",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show(truncate=False) + +-----------------------------------+ + |url_encode(url) | + +-----------------------------------+ + |%7E%21%40%23%24%25%5E%26*%28%29_%2B| + +-----------------------------------+ """ - fn = "theta_sketch_agg" - if lgNomEntries is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) + return _invoke_function_over_columns("url_encode", str) + + +# ---------------------- Misc Functions ---------------------- @_try_remote_functions -def theta_union_agg( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, -) -> Column: +def input_file_name() -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - ThetaSketch that is the union of the Theta sketches in the input column. + Creates a string column for the file name of the current Spark task. - .. versionadded:: 4.1.0 + .. versionadded:: 1.6.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries for the union operation - (must be between 4 and 26, defaults to 12) + .. versionchanged:: 3.4.0 + Supports Spark Connect. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged ThetaSketch. + file names. See Also -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + :meth:`pyspark.sql.functions.input_file_block_length` + :meth:`pyspark.sql.functions.input_file_block_start` Examples -------- + >>> import os >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,2,3], "INT") - >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df2 = spark.createDataFrame([4,5,5,6], "INT") - >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.theta_sketch_estimate(sf.theta_union_agg("sketch"))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_union_agg(sketch, 12))| - +--------------------------------------------------+ - | 6| - +--------------------------------------------------+ + >>> path = os.path.abspath(__file__) + >>> df = spark.read.text(path) + >>> df.select(sf.input_file_name()).first() + Row(input_file_name()='file:///...') """ - fn = "theta_union_agg" - if lgNomEntries is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) + return _invoke_function("input_file_name") @_try_remote_functions -def theta_intersection_agg(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - ThetaSketch that is the intersection of the Theta sketches in the input column +def monotonically_increasing_id() -> Column: + """A column that generates monotonically increasing 64-bit integers. - .. versionadded:: 4.1.0 + The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. + The current implementation puts the partition ID in the upper 31 bits, and the record number + within each partition in the lower 33 bits. The assumption is that the data frame has + less than 1 billion partitions, and each partition has less than 8 billion records. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The function is non-deterministic because its result depends on partition IDs. + + As an example, consider a :class:`DataFrame` with two partitions, each with 3 records. + This expression would return the following IDs: + 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected ThetaSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.theta_intersection` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + last value of the group. Examples -------- >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,2,3], "INT") - >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df2 = spark.createDataFrame([2,3,3,4], "INT") - >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.theta_sketch_estimate(sf.theta_intersection_agg("sketch"))).show() - +-----------------------------------------------------+ - |theta_sketch_estimate(theta_intersection_agg(sketch))| - +-----------------------------------------------------+ - | 2| - +-----------------------------------------------------+ + >>> spark.range(0, 10, 1, 2).select( + ... "*", + ... sf.spark_partition_id(), + ... sf.monotonically_increasing_id()).show() + +---+--------------------+-----------------------------+ + | id|SPARK_PARTITION_ID()|monotonically_increasing_id()| + +---+--------------------+-----------------------------+ + | 0| 0| 0| + | 1| 0| 1| + | 2| 0| 2| + | 3| 0| 3| + | 4| 0| 4| + | 5| 1| 8589934592| + | 6| 1| 8589934593| + | 7| 1| 8589934594| + | 8| 1| 8589934595| + | 9| 1| 8589934596| + +---+--------------------+-----------------------------+ """ - fn = "theta_intersection_agg" - return _invoke_function_over_columns(fn, col) + return _invoke_function("monotonically_increasing_id") @_try_remote_functions -def tuple_sketch_agg_double( - key: "ColumnOrName", - summary: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch with double summaries built from the key and summary columns. +def spark_partition_id() -> Column: + """A column for partition ID. - .. versionadded:: 4.2.0 + .. versionadded:: 1.6.0 - Parameters - ---------- - key : :class:`~pyspark.sql.Column` or column name - The column containing key values. - A column that evaluates to an array, binary, double, float, integer, long, or string. - summary : :class:`~pyspark.sql.Column` or column name - The column containing double summary values. - A column that evaluates to a double. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + This is non deterministic because it depends on data partitioning and task scheduling. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` - :meth:`pyspark.sql.functions.tuple_sketch_summary_double` - :meth:`pyspark.sql.functions.tuple_union_agg_double` + partition id the record belongs to. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +--------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| - +--------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(10, numPartitions=5).select("*", sf.spark_partition_id()).show() + +---+--------------------+ + | id|SPARK_PARTITION_ID()| + +---+--------------------+ + | 0| 0| + | 1| 0| + | 2| 1| + | 3| 1| + | 4| 2| + | 5| 2| + | 6| 3| + | 7| 3| + | 8| 4| + | 9| 4| + +---+--------------------+ """ - fn = "tuple_sketch_agg_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) + return _invoke_function("spark_partition_id") @_try_remote_functions -def tuple_sketch_agg_integer( - key: "ColumnOrName", - summary: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch with integer summaries built from the key and summary columns. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - key : :class:`~pyspark.sql.Column` or column name - The column containing key values. - A column that evaluates to an array, binary, double, float, integer, long, or string. - summary : :class:`~pyspark.sql.Column` or column name - The column containing integer summary values. - A column that evaluates to an integer. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" +def current_catalog() -> Column: + """Returns the current catalog. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the TupleSketch. + .. versionadded:: 3.5.0 See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` - :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` - :meth:`pyspark.sql.functions.tuple_union_agg_integer` + :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_schema` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +----------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| - +----------------------------------------------------------------------------+ - | 2.0| - +----------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.current_catalog()).show() + +-----------------+ + |current_catalog()| + +-----------------+ + | spark_catalog| + +-----------------+ """ - fn = "tuple_sketch_agg_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) + return _invoke_function("current_catalog") @_try_remote_functions -def tuple_union_agg_double( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the union of the double TupleSketch objects in the input column. +def current_path() -> Column: + """Returns the current SQL path as a comma-separated list of qualified schema names. .. versionadded:: 4.2.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" - - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. - See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_double` + :meth:`pyspark.sql.functions.current_catalog` + :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_schema` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(3, 30.0), (4, 40.0)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_union_agg_double("sketch"))).show() - +---------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_union_agg_double(sketch, 12, sum))| - +---------------------------------------------------------------------+ - | 4.0| - +---------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.current_path()).show() # doctest: +SKIP + +----------------------------------------------------+ + | current_path()| + +----------------------------------------------------+ + |system.builtin,system.session,spark_catalog.default | + +----------------------------------------------------+ """ - fn = "tuple_union_agg_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) + return _invoke_function("current_path") @_try_remote_functions -def tuple_union_agg_integer( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the union of the integer TupleSketch objects in the input column. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" +def current_database() -> Column: + """Returns the current database. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. + .. versionadded:: 3.5.0 See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_integer` + :meth:`pyspark.sql.functions.current_catalog` + :meth:`pyspark.sql.functions.current_schema` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(3, 30), (4, 40)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_union_agg_integer("sketch"))).show() - +-----------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch, 12, sum))| - +-----------------------------------------------------------------------+ - | 4.0| - +-----------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.current_database()).show() + +----------------+ + |current_schema()| + +----------------+ + | default| + +----------------+ """ - fn = "tuple_union_agg_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) + return _invoke_function("current_database") @_try_remote_functions -def tuple_intersection_agg_double( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def current_schema() -> Column: + """Returns the current database. + + .. versionadded:: 3.5.0 + + See Also + -------- + :meth:`pyspark.sql.functions.current_catalog` + :meth:`pyspark.sql.functions.current_database` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.current_schema()).show() + +----------------+ + |current_schema()| + +----------------+ + | default| + +----------------+ """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the intersection of the double TupleSketch objects in the input column. + return _invoke_function("current_schema") - .. versionadded:: 4.2.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" +@_try_remote_functions +def current_user() -> Column: + """Returns the current database. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + .. versionadded:: 3.5.0 See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_intersection_double` + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.session_user` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0), (3, 30.0)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(2, 40.0), (3, 50.0), (4, 60.0)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_intersection_agg_double("sketch"))).show() - +------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch, sum))| - +------------------------------------------------------------------------+ - | 2.0| - +------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.current_user()).show() # doctest: +SKIP + +--------------+ + |current_user()| + +--------------+ + | ruifeng.zheng| + +--------------+ """ - fn = "tuple_intersection_agg_double" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + return _invoke_function("current_user") @_try_remote_functions -def tuple_intersection_agg_integer( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def user() -> Column: + """Returns the current database. + + .. versionadded:: 3.5.0 + + See Also + -------- + :meth:`pyspark.sql.functions.current_user` + :meth:`pyspark.sql.functions.session_user` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.user()).show() # doctest: +SKIP + +--------------+ + | user()| + +--------------+ + | ruifeng.zheng| + +--------------+ """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the intersection of the integer TupleSketch objects in the input column. + return _invoke_function("user") - .. versionadded:: 4.2.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" +@_try_remote_functions +def session_user() -> Column: + """Returns the user name of current execution context. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + .. versionadded:: 4.0.0 See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_intersection_integer` + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.current_user` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10), (2, 20), (3, 30)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(2, 40), (3, 50), (4, 60)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_agg_integer("sketch"))).show() - +--------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch, sum))| - +--------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.session_user()).show() # doctest: +SKIP + +--------------+ + |session_user()| + +--------------+ + | ruifeng.zheng| + +--------------+ """ - fn = "tuple_intersection_agg_integer" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + return _invoke_function("session_user") @_try_remote_functions -def kll_sketch_agg_bigint( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - KllLongsSketch built with the values in the input column. The optional k parameter - controls the size and accuracy of the sketch (default 200, range 8-65535). +def uuid(seed: Optional[Union[Column, int]] = None) -> Column: + """Returns an universally unique identifier (UUID) string. + The value is returned as a canonical UUID 36-character string. .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing bigint values to aggregate. - A column that evaluates to an integral. - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (default 200, range 8-65535) - A column that evaluates to an integer. Must be a constant. - - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the KllLongsSketch. + seed : :class:`~pyspark.sql.Column` or int + Optional random number seed to use. Examples -------- + Example 1: Generate UUIDs with random seed + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> result = df.agg(sf.kll_sketch_agg_bigint("value")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> spark.range(5).select(sf.uuid()).show(truncate=False) # doctest: +SKIP + +------------------------------------+ + |uuid() | + +------------------------------------+ + |627ae05e-b319-42b5-b4e4-71c8c9754dd1| + |f781cce5-a2e2-464d-bc8b-426ff448e404| + |15e2e66e-8416-4ea2-af3c-409363408189| + |fb1d6178-7676-4791-baa9-f2ddcc494515| + |d48665e8-2657-4c6b-b7c8-8ae0cd646e41| + +------------------------------------+ + + Example 2: Generate UUIDs with a specified seed + + >>> from pyspark.sql import functions as sf + >>> spark.range(0, 5, 1, 1).select(sf.uuid(seed=123)).show(truncate=False) + +------------------------------------+ + |uuid() | + +------------------------------------+ + |4c99192d-23d6-4d88-b814-a634398120f0| + |af506873-3c53-41e3-8354-a24856b8de8a| + |7b4b370e-e867-47e2-93c0-f6990463a12d| + |1c4d1733-ff1a-4a6c-b144-0b0345adf0d0| + |7478f235-f8bc-4112-8e59-a28f50e46890| + +------------------------------------+ """ - fn = "kll_sketch_agg_bigint" - if k is None: - return _invoke_function_over_columns(fn, col) + from pyspark.sql.classic.column import _to_java_column + + if seed is None: + return _invoke_function("uuid") else: - return _invoke_function_over_columns(fn, col, lit(k)) + return _invoke_function("uuid", _to_java_column(lit(seed))) @_try_remote_functions -def kll_sketch_agg_float( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, -) -> Column: +def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None) -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - KllFloatsSketch built with the values in the input column. The optional k parameter - controls the size and accuracy of the sketch (default 200, range 8-65535). + Returns `null` if the input column is `true`; throws an exception + with the provided error message otherwise. - .. versionadded:: 4.1.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing float values to aggregate - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (default 200, range 8-65535) - A column that evaluates to an integer. Must be a constant. + column name or column that represents the input column to test. + A column that evaluates to a boolean. + errMsg : :class:`~pyspark.sql.Column` or literal string, optional + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the KllFloatsSketch. + `null` if the input column is `true` otherwise throws an error with specified message. + Returns a column that always evaluates to NULL. + + See Also + -------- + :meth:`pyspark.sql.functions.raise_error` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> result = df.agg(sf.kll_sketch_agg_float("value")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(0, 1)], ['a', 'b']) + >>> df.select('*', sf.assert_true(df.a < df.b)).show() + +---+---+--------------------------------------------+ + | a| b|assert_true((a < b), '(a < b)' is not true!)| + +---+---+--------------------------------------------+ + | 0| 1| NULL| + +---+---+--------------------------------------------+ + + >>> df.select('*', sf.assert_true(df.a < df.b, df.a)).show() + +---+---+-----------------------+ + | a| b|assert_true((a < b), a)| + +---+---+-----------------------+ + | 0| 1| NULL| + +---+---+-----------------------+ + + >>> df.select('*', sf.assert_true(df.a < df.b, 'error')).show() + +---+---+---------------------------+ + | a| b|assert_true((a < b), error)| + +---+---+---------------------------+ + | 0| 1| NULL| + +---+---+---------------------------+ + + >>> df.select('*', sf.assert_true(df.a > df.b, 'My error msg')).show() # doctest: +SKIP + ... + java.lang.RuntimeException: My error msg + ... """ - fn = "kll_sketch_agg_float" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + errMsg = _enum_to_value(errMsg) + if errMsg is None: + return _invoke_function_over_columns("assert_true", col) + if not isinstance(errMsg, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "errMsg", + "arg_type": type(errMsg).__name__, + }, + ) + return _invoke_function_over_columns("assert_true", col, lit(errMsg)) @_try_remote_functions -def kll_sketch_agg_double( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, -) -> Column: +def raise_error(errMsg: Union[Column, str]) -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - KllDoublesSketch built with the values in the input column. The optional k parameter - controls the size and accuracy of the sketch (default 200, range 8-65535). + Throws an exception with the provided error message. - .. versionadded:: 4.1.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing double values to aggregate. - A column that evaluates to a float or double. - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (default 200, range 8-65535) - A column that evaluates to an integer. Must be a constant. + errMsg : :class:`~pyspark.sql.Column` or literal string + A Python string literal or column containing the error message. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the KllDoublesSketch. + throws an error with specified message. + Returns a column that always evaluates to NULL. + + See Also + -------- + :meth:`pyspark.sql.functions.assert_true` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> result = df.agg(sf.kll_sketch_agg_double("value")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.raise_error("My error message")).show() # doctest: +SKIP + ... + java.lang.RuntimeException: My error message + ... """ - fn = "kll_sketch_agg_double" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + errMsg = _enum_to_value(errMsg) + if not isinstance(errMsg, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "errMsg", + "arg_type": type(errMsg).__name__, + }, + ) + return _invoke_function_over_columns("raise_error", lit(errMsg)) @_try_remote_functions -def kll_merge_agg_bigint( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, +def hmac( + key: "ColumnOrName", + message: "ColumnOrName", + algorithm: Optional["ColumnOrName"] = None, ) -> Column: """ - Aggregate function: merges binary KllLongsSketch representations and returns the - merged sketch. The optional k parameter controls the size and accuracy of the merged - sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value - from the first input sketch. + Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the + given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with :func:`hex` or + :func:`base64` for a textual value. The default algorithm is 'SHA-256'. - .. versionadded:: 4.1.2 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary KllLongsSketch representations - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (range 8-65535) - A column that evaluates to an integer. Must be a constant. + key : :class:`~pyspark.sql.Column` or column name + The secret key, as a binary value. + message : :class:`~pyspark.sql.Column` or column name + The message to authenticate, as a binary value. + algorithm : :class:`~pyspark.sql.Column` or column name, optional + The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. + The default is SHA-256. Returns ------- :class:`~pyspark.sql.Column` - The merged binary representation of the KllLongsSketch. + A new column that contains the raw HMAC bytes. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,3], "INT") - >>> df2 = spark.createDataFrame([4,5,6], "INT") - >>> sketch1 = df1.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch2 = df2.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_bigint("sketch").alias("merged")) - >>> n = merged.select(sf.kll_sketch_get_n_bigint("merged")).first()[0] - >>> n - 6 + + Example 1: Compute the HMAC with the default SHA-256 algorithm. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) + >>> df.select(sf.hex(sf.hmac(df.key, df.message))).show(truncate=False) + +----------------------------------------------------------------+ + |hex(hmac(key, message, SHA-256)) | + +----------------------------------------------------------------+ + |6E9EF29B75FFFC5B7ABAE527D58FDADB2FE42E7219011976917343065F58ED4A| + +----------------------------------------------------------------+ + + Example 2: Compute the HMAC with an explicit algorithm. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) + >>> df.select(sf.hex(sf.hmac(df.key, df.message, sf.lit("SHA-1")))).show(truncate=False) + +----------------------------------------+ + |hex(hmac(key, message, SHA-1)) | + +----------------------------------------+ + |2088DF74D5F2146B48146CAF4965377E9D0BE3A4| + +----------------------------------------+ """ - fn = "kll_merge_agg_bigint" - if k is None: - return _invoke_function_over_columns(fn, col) + if algorithm is None: + return _invoke_function_over_columns("hmac", key, message) else: - return _invoke_function_over_columns(fn, col, lit(k)) + return _invoke_function_over_columns("hmac", key, message, algorithm) @_try_remote_functions -def kll_merge_agg_float( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, +def aes_encrypt( + input: "ColumnOrName", + key: "ColumnOrName", + mode: Optional["ColumnOrName"] = None, + padding: Optional["ColumnOrName"] = None, + iv: Optional["ColumnOrName"] = None, + aad: Optional["ColumnOrName"] = None, ) -> Column: """ - Aggregate function: merges binary KllFloatsSketch representations and returns the - merged sketch. The optional k parameter controls the size and accuracy of the merged - sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value - from the first input sketch. + Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. + Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, + `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization + vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 + bytes for GCM. If not provided, a random vector will be generated and prepended to the + output. Optional additional authenticated data (AAD) is only supported for GCM. If provided + for encryption, the identical AAD value must be provided for decryption. The default mode is + GCM. - .. versionadded:: 4.1.2 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary KllFloatsSketch representations - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (range 8-65535) - A column that evaluates to an integer. Must be a constant. + input : :class:`~pyspark.sql.Column` or column name + The binary value to encrypt. + A column that evaluates to a binary. + key : :class:`~pyspark.sql.Column` or column name + The passphrase to use to encrypt the data. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or str, optional + Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + GCM, CBC. + A column that evaluates to a string. + padding : :class:`~pyspark.sql.Column` or column name, optional + Specifies how to pad messages whose length is not a multiple of the block size. Valid + values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + for CBC. + A column that evaluates to a string. + iv : :class:`~pyspark.sql.Column` or column name, optional + Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + "". 16-byte array for CBC mode. 12-byte array for GCM mode. + A column that evaluates to a binary. + aad : :class:`~pyspark.sql.Column` or column name, optional + Optional additional authenticated data. Only supported for GCM mode. This can be any + free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The merged binary representation of the KllFloatsSketch. + A new column that contains an encrypted value. + Returns a column that evaluates to a binary. + + See Also + -------- + :meth:`pyspark.sql.functions.aes_decrypt` + :meth:`pyspark.sql.functions.try_aes_decrypt` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "FLOAT") - >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "FLOAT") - >>> sketch1 = df1.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch2 = df2.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_float("sketch").alias("merged")) - >>> n = merged.select(sf.kll_sketch_get_n_float("merged")).first()[0] - >>> n - 6 + + Example 1: Encrypt data with key, mode, padding, iv and aad. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "000000000000000000000000", "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "iv", "aad"] + ... ) + >>> df.select(sf.base64(sf.aes_encrypt( + ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex")), df.aad) + ... )).show(truncate=False) + +-----------------------------------------------------------------------+ + |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), aad))| + +-----------------------------------------------------------------------+ + |AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4 | + +-----------------------------------------------------------------------+ + + Example 2: Encrypt data with key, mode, padding and iv. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "000000000000000000000000", "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "iv", "aad"] + ... ) + >>> df.select(sf.base64(sf.aes_encrypt( + ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex"))) + ... )).show(truncate=False) + +--------------------------------------------------------------------+ + |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), ))| + +--------------------------------------------------------------------+ + |AAAAAAAAAAAAAAAAQiYi+sRNYDAOTjdSEcYBFsAWPL1f | + +--------------------------------------------------------------------+ + + Example 3: Encrypt data with key, mode and padding. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark SQL", "1234567890abcdef", "ECB", "PKCS",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode", df.padding), + ... df.key, df.mode, df.padding + ... ).cast("STRING")).show(truncate=False) + +---------------------------------------------------------------------------------------------+ + |CAST(aes_decrypt(aes_encrypt(input, key, mode, padding, , ), key, mode, padding, ) AS STRING)| + +---------------------------------------------------------------------------------------------+ + |Spark SQL | + +---------------------------------------------------------------------------------------------+ + + Example 4: Encrypt data with key and mode. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark SQL", "0000111122223333", "ECB",)], + ... ["input", "key", "mode"] + ... ) + >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode"), + ... df.key, df.mode + ... ).cast("STRING")).show(truncate=False) + +---------------------------------------------------------------------------------------------+ + |CAST(aes_decrypt(aes_encrypt(input, key, mode, DEFAULT, , ), key, mode, DEFAULT, ) AS STRING)| + +---------------------------------------------------------------------------------------------+ + |Spark SQL | + +---------------------------------------------------------------------------------------------+ + + Example 5: Encrypt data with key. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark SQL", "abcdefghijklmnop",)], + ... ["input", "key"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(sf.base64(sf.aes_encrypt(df.input, df.key))), df.key + ... ).cast("STRING")).show(truncate=False) + +-------------------------------------------------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(base64(aes_encrypt(input, key, GCM, DEFAULT, , ))), key, GCM, DEFAULT, ) AS STRING)| + +-------------------------------------------------------------------------------------------------------------+ + |Spark SQL | + +-------------------------------------------------------------------------------------------------------------+ """ - fn = "kll_merge_agg_float" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + _mode = lit("GCM") if mode is None else mode + _padding = lit("DEFAULT") if padding is None else padding + _iv = lit("") if iv is None else iv + _aad = lit("") if aad is None else aad + return _invoke_function_over_columns("aes_encrypt", input, key, _mode, _padding, _iv, _aad) @_try_remote_functions -def kll_merge_agg_double( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, +def aes_decrypt( + input: "ColumnOrName", + key: "ColumnOrName", + mode: Optional["ColumnOrName"] = None, + padding: Optional["ColumnOrName"] = None, + aad: Optional["ColumnOrName"] = None, ) -> Column: """ - Aggregate function: merges binary KllDoublesSketch representations and returns the - merged sketch. The optional k parameter controls the size and accuracy of the merged - sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value - from the first input sketch. + Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, + 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', + 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is + only supported for GCM. If provided for encryption, the identical AAD value must be provided + for decryption. The default mode is GCM. - .. versionadded:: 4.1.2 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary KllDoublesSketch representations - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (range 8-65535) - A column that evaluates to an integer. Must be a constant. + input : :class:`~pyspark.sql.Column` or column name + The binary value to decrypt. + A column that evaluates to a binary. + key : :class:`~pyspark.sql.Column` or column name + The passphrase to use to decrypt the data. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or column name, optional + Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + GCM, CBC. + A column that evaluates to a string. + padding : :class:`~pyspark.sql.Column` or column name, optional + Specifies how to pad messages whose length is not a multiple of the block size. Valid + values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + for CBC. + A column that evaluates to a string. + aad : :class:`~pyspark.sql.Column` or column name, optional + Optional additional authenticated data. Only supported for GCM mode. This can be any + free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The merged binary representation of the KllDoublesSketch. + A new column that contains a decrypted value. + Returns a column that evaluates to a binary. + + See Also + -------- + :meth:`pyspark.sql.functions.aes_encrypt` + :meth:`pyspark.sql.functions.try_aes_decrypt` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "DOUBLE") - >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "DOUBLE") - >>> sketch1 = df1.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch2 = df2.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_double("sketch").alias("merged")) - >>> n = merged.select(sf.kll_sketch_get_n_double("merged")).first()[0] - >>> n - 6 + + Example 1: Decrypt data with key, mode, padding and aad. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", + ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "aad"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad + ... ).cast("STRING")).show(truncate=False) + +---------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| + +---------------------------------------------------------------------+ + |Spark | + +---------------------------------------------------------------------+ + + Example 2: Decrypt data with key, mode and padding. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding + ... ).cast("STRING")).show(truncate=False) + +------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| + +------------------------------------------------------------------+ + |Spark | + +------------------------------------------------------------------+ + + Example 3: Decrypt data with key and mode. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode" + ... ).cast("STRING")).show(truncate=False) + +------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| + +------------------------------------------------------------------+ + |Spark | + +------------------------------------------------------------------+ + + Example 4: Decrypt data with key. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", + ... "0000111122223333",)], + ... ["input", "key"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unhex(df.input), df.key + ... ).cast("STRING")).show(truncate=False) + +--------------------------------------------------------------+ + |CAST(aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| + +--------------------------------------------------------------+ + |Spark | + +--------------------------------------------------------------+ """ - fn = "kll_merge_agg_double" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + _mode = lit("GCM") if mode is None else mode + _padding = lit("DEFAULT") if padding is None else padding + _aad = lit("") if aad is None else aad + return _invoke_function_over_columns("aes_decrypt", input, key, _mode, _padding, _aad) @_try_remote_functions -def kll_sketch_to_string_bigint(col: "ColumnOrName") -> Column: +def try_aes_decrypt( + input: "ColumnOrName", + key: "ColumnOrName", + mode: Optional["ColumnOrName"] = None, + padding: Optional["ColumnOrName"] = None, + aad: Optional["ColumnOrName"] = None, +) -> Column: """ - Returns a string with human readable summary information about the KLL bigint sketch. + This is a special version of `aes_decrypt` that performs the same operation, + but returns a NULL value instead of raising an error if the decryption cannot be performed. + Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, + 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', + 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is + only supported for GCM. If provided for encryption, the identical AAD value must be provided + for decryption. The default mode is GCM. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. + input : :class:`~pyspark.sql.Column` or column name + The binary value to decrypt. + A column that evaluates to a binary. + key : :class:`~pyspark.sql.Column` or column name + The passphrase to use to decrypt the data. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or column name, optional + Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + GCM, CBC. + A column that evaluates to a string. + padding : :class:`~pyspark.sql.Column` or column name, optional + Specifies how to pad messages whose length is not a multiple of the block size. Valid + values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + for CBC. + A column that evaluates to a string. + aad : :class:`~pyspark.sql.Column` or column name, optional + Optional additional authenticated data. Only supported for GCM mode. This can be any + free-form input and must be provided for both encryption and decryption. A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - A string representation of the sketch. - Returns a column that evaluates to a string. + A new column that contains a decrypted value or a NULL value. + Returns a column that evaluates to a binary. + + See Also + -------- + :meth:`pyspark.sql.functions.aes_encrypt` + :meth:`pyspark.sql.functions.aes_decrypt` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_to_string_bigint("sketch")).first()[0] - >>> "kll" in result.lower() - True + + Example 1: Decrypt data with key, mode, padding and aad. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", + ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "aad"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad + ... ).cast("STRING")).show(truncate=False) + +-------------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| + +-------------------------------------------------------------------------+ + |Spark | + +-------------------------------------------------------------------------+ + + Example 2: Failed to decrypt data with key, mode, padding and aad. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT", + ... "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "aad"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad + ... ).cast("STRING")).show(truncate=False) + +-------------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| + +-------------------------------------------------------------------------+ + |NULL | + +-------------------------------------------------------------------------+ + + Example 3: Decrypt data with key, mode and padding. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding + ... ).cast("STRING")).show(truncate=False) + +----------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| + +----------------------------------------------------------------------+ + |Spark | + +----------------------------------------------------------------------+ + + Example 4: Decrypt data with key and mode. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode" + ... ).cast("STRING")).show(truncate=False) + +----------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| + +----------------------------------------------------------------------+ + |Spark | + +----------------------------------------------------------------------+ + + Example 5: Decrypt data with key. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", + ... "0000111122223333",)], + ... ["input", "key"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unhex(df.input), df.key + ... ).cast("STRING")).show(truncate=False) + +------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| + +------------------------------------------------------------------+ + |Spark | + +------------------------------------------------------------------+ """ - fn = "kll_sketch_to_string_bigint" - return _invoke_function_over_columns(fn, col) + _mode = lit("GCM") if mode is None else mode + _padding = lit("DEFAULT") if padding is None else padding + _aad = lit("") if aad is None else aad + return _invoke_function_over_columns("try_aes_decrypt", input, key, _mode, _padding, _aad) @_try_remote_functions -def kll_sketch_to_string_float(col: "ColumnOrName") -> Column: +def input_file_block_length() -> Column: """ - Returns a string with human readable summary information about the KLL float sketch. - - .. versionadded:: 4.1.0 + Returns the length of the block being read, or -1 if not available. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. + .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - A string representation of the sketch. - Returns a column that evaluates to a string. + See Also + -------- + :meth:`pyspark.sql.functions.input_file_name` + :meth:`pyspark.sql.functions.input_file_block_start` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_to_string_float("sketch")).first()[0] - >>> "kll" in result.lower() - True + >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") + >>> df.select(sf.input_file_block_length()).show() + +-------------------------+ + |input_file_block_length()| + +-------------------------+ + | 87| + | 87| + | 87| + | 87| + | 87| + | 87| + | 87| + | 87| + +-------------------------+ """ - fn = "kll_sketch_to_string_float" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_columns("input_file_block_length") @_try_remote_functions -def kll_sketch_to_string_double(col: "ColumnOrName") -> Column: +def input_file_block_start() -> Column: """ - Returns a string with human readable summary information about the KLL double sketch. - - .. versionadded:: 4.1.0 + Returns the start offset of the block being read, or -1 if not available. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. + .. versionadded:: 3.5.0 - Returns - ------- - :class:`~pyspark.sql.Column` - A string representation of the sketch. - Returns a column that evaluates to a string. + See Also + -------- + :meth:`pyspark.sql.functions.input_file_name` + :meth:`pyspark.sql.functions.input_file_block_length` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_to_string_double("sketch")).first()[0] - >>> "kll" in result.lower() - True + >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") + >>> df.select(sf.input_file_block_start()).show() + +------------------------+ + |input_file_block_start()| + +------------------------+ + | 0| + | 0| + | 0| + | 0| + | 0| + | 0| + | 0| + | 0| + +------------------------+ """ - fn = "kll_sketch_to_string_double" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_columns("input_file_block_start") @_try_remote_functions -def kll_sketch_get_n_bigint(col: "ColumnOrName") -> Column: +def reflect(*cols: "ColumnOrName") -> Column: """ - Returns the number of items collected in the KLL bigint sketch. + Calls a method with reflection. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. - A column that evaluates to a binary. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a Column representing literal string for the class name, + and the second element should be a Column representing literal string for the method name, + and the remaining are input arguments (Columns or column names) to the Java method. - Returns - ------- - :class:`~pyspark.sql.Column` - The count of items in the sketch. - Returns a column that evaluates to a long. + See Also + -------- + :meth:`pyspark.sql.functions.java_method` + :meth:`pyspark.sql.functions.try_reflect` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_n_bigint("sketch")).show() - +-------------------------------+ - |kll_sketch_get_n_bigint(sketch)| - +-------------------------------+ - | 5| - +-------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) + >>> df.select( + ... sf.reflect(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') + ... ).show(truncate=False) + +--------------------------------------+ + |reflect(java.util.UUID, fromString, a)| + +--------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +--------------------------------------+ """ - fn = "kll_sketch_get_n_bigint" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_seq_of_columns("reflect", cols) @_try_remote_functions -def kll_sketch_get_n_float(col: "ColumnOrName") -> Column: +def java_method(*cols: "ColumnOrName") -> Column: """ - Returns the number of items collected in the KLL float sketch. + Calls a method with reflection. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a Column representing literal string for the class name, + and the second element should be a Column representing literal string for the method name, + and the remaining are input arguments (Columns or column names) to the Java method. - Returns - ------- - :class:`~pyspark.sql.Column` - The count of items in the sketch. - Returns a column that evaluates to a long. + See Also + -------- + :meth:`pyspark.sql.functions.reflect` + :meth:`pyspark.sql.functions.try_reflect` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_n_float("sketch")).show() - +------------------------------+ - |kll_sketch_get_n_float(sketch)| - +------------------------------+ - | 5| - +------------------------------+ - """ - fn = "kll_sketch_get_n_float" - return _invoke_function_over_columns(fn, col) - - -@_try_remote_functions -def kll_sketch_get_n_double(col: "ColumnOrName") -> Column: - """ - Returns the number of items collected in the KLL double sketch. - - .. versionadded:: 4.1.0 + Example 1: Reflecting a method call with a column argument - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select( + ... sf.java_method( + ... sf.lit("java.util.UUID"), + ... sf.lit("fromString"), + ... sf.lit("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2") + ... ) + ... ).show(truncate=False) + +-----------------------------------------------------------------------------+ + |java_method(java.util.UUID, fromString, a5cf6c42-0c85-418f-af6c-3e4e5b1328f2)| + +-----------------------------------------------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +-----------------------------------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - The count of items in the sketch. - Returns a column that evaluates to a long. + Example 2: Reflecting a method call with a column name argument - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_n_double("sketch")).show() - +-------------------------------+ - |kll_sketch_get_n_double(sketch)| - +-------------------------------+ - | 5| - +-------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) + >>> df.select( + ... sf.java_method(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') + ... ).show(truncate=False) + +------------------------------------------+ + |java_method(java.util.UUID, fromString, a)| + +------------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +------------------------------------------+ """ - fn = "kll_sketch_get_n_double" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_seq_of_columns("java_method", cols) @_try_remote_functions -def kll_sketch_merge_bigint(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def try_reflect(*cols: "ColumnOrName") -> Column: """ - Merges two KLL bigint sketch buffers together into one. + This is a special version of `reflect` that performs the same operation, but returns a NULL + value instead of raising an error if the invoke method thrown exception. - .. versionadded:: 4.1.0 + + .. versionadded:: 4.0.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The first KLL bigint sketch. - A column that evaluates to a binary. - right : :class:`~pyspark.sql.Column` or column name - The second KLL bigint sketch. - A column that evaluates to a binary. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a Column representing literal string for the class name, + and the second element should be a Column representing literal string for the method name, + and the remaining are input arguments (Columns or column names) to the Java method. - Returns - ------- - :class:`~pyspark.sql.Column` - The merged KLL sketch. - Returns a column that evaluates to a binary. + See Also + -------- + :meth:`pyspark.sql.functions.reflect` + :meth:`pyspark.sql.functions.java_method` Examples -------- + Example 1: Reflecting a method call with arguments + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_merge_bigint("sketch", "sketch")).first()[0] - >>> result is not None and len(result) > 0 - True - """ - fn = "kll_sketch_merge_bigint" - return _invoke_function_over_columns(fn, left, right) + >>> df = spark.createDataFrame([("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2",)], ["a"]) + >>> df.select( + ... sf.try_reflect(sf.lit("java.util.UUID"), sf.lit("fromString"), "a") + ... ).show(truncate=False) + +------------------------------------------+ + |try_reflect(java.util.UUID, fromString, a)| + +------------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +------------------------------------------+ + Example 2: Exception in the reflection call, resulting in null -@_try_remote_functions -def kll_sketch_merge_float(left: "ColumnOrName", right: "ColumnOrName") -> Column: + >>> from pyspark.sql import functions as sf + >>> spark.range(1).select( + ... sf.try_reflect(sf.lit("scala.Predef"), sf.lit("require"), sf.lit(False)) + ... ).show(truncate=False) + +-----------------------------------------+ + |try_reflect(scala.Predef, require, false)| + +-----------------------------------------+ + |NULL | + +-----------------------------------------+ """ - Merges two KLL float sketch buffers together into one. + return _invoke_function_over_seq_of_columns("try_reflect", cols) - .. versionadded:: 4.1.0 - Parameters - ---------- - left : :class:`~pyspark.sql.Column` or column name - The first KLL float sketch. - A column that evaluates to a binary. - right : :class:`~pyspark.sql.Column` or column name - The second KLL float sketch. - A column that evaluates to a binary. +@_try_remote_functions +def version() -> Column: + """ + Returns the Spark version. The string contains 2 fields, the first being a release version + and the second being a git revision. - Returns - ------- - :class:`~pyspark.sql.Column` - The merged KLL sketch. - Returns a column that evaluates to a binary. + .. versionadded:: 3.5.0 Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_merge_float("sketch", "sketch")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> spark.range(1).select(sf.version()).show(truncate=False) # doctest: +SKIP + +----------------------------------------------+ + |version() | + +----------------------------------------------+ + |4.0.0 4f8d1f575e99aeef8990c63a9614af0fc5479330| + +----------------------------------------------+ """ - fn = "kll_sketch_merge_float" - return _invoke_function_over_columns(fn, left, right) + return _invoke_function_over_columns("version") @_try_remote_functions -def kll_sketch_merge_double(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def typeof(col: "ColumnOrName") -> Column: """ - Merges two KLL double sketch buffers together into one. + Return DDL-formatted type string for the data type of the input. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The first KLL double sketch. - A column that evaluates to a binary. - right : :class:`~pyspark.sql.Column` or column name - The second KLL double sketch. - A column that evaluates to a binary. - - Returns - ------- - :class:`~pyspark.sql.Column` - The merged KLL sketch. - Returns a column that evaluates to a binary. + col : :class:`~pyspark.sql.Column` or column name Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_merge_double("sketch", "sketch")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> df = spark.createDataFrame([(True, 1, 1.0, 'xyz',)], ['a', 'b', 'c', 'd']) + >>> df.select(sf.typeof(df.a), sf.typeof(df.b), sf.typeof('c'), sf.typeof('d')).show() + +---------+---------+---------+---------+ + |typeof(a)|typeof(b)|typeof(c)|typeof(d)| + +---------+---------+---------+---------+ + | boolean| bigint| double| string| + +---------+---------+---------+---------+ """ - fn = "kll_sketch_merge_double" - return _invoke_function_over_columns(fn, left, right) + return _invoke_function_over_columns("typeof", col) @_try_remote_functions -def kll_sketch_get_quantile_bigint(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: +def bitmap_bit_position(col: "ColumnOrName") -> Column: """ - Extracts a quantile value from a KLL bigint sketch given an input rank value. - The rank can be a single value or an array. + Returns the bit position for the given input column. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. - A column that evaluates to a binary. - rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0). - A column that evaluates to a double or array. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + The input column. + A column that evaluates to a long. - Returns - ------- - :class:`~pyspark.sql.Column` - The quantile value(s). - Returns a column that evaluates to a long, or an array of longs if the rank - argument is an array. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_quantile_bigint("sketch", sf.lit(0.5))).show() - +-------------------------------------------+ - |kll_sketch_get_quantile_bigint(sketch, 0.5)| - +-------------------------------------------+ - | 3| - +-------------------------------------------+ + >>> df = spark.createDataFrame([(123,)], ['a']) + >>> df.select('*', sf.bitmap_bit_position('a')).show() + +---+----------------------+ + | a|bitmap_bit_position(a)| + +---+----------------------+ + |123| 122| + +---+----------------------+ """ - fn = "kll_sketch_get_quantile_bigint" - return _invoke_function_over_columns(fn, sketch, rank) + return _invoke_function_over_columns("bitmap_bit_position", col) @_try_remote_functions -def kll_sketch_get_quantile_float(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: +def bitmap_bucket_number(col: "ColumnOrName") -> Column: """ - Extracts a quantile value from a KLL float sketch given an input rank value. - The rank can be a single value or an array. + Returns the bucket number for the given input column. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. - rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0). - A column that evaluates to a double or array. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + The input column. + A column that evaluates to a long. - Returns - ------- - :class:`~pyspark.sql.Column` - The quantile value(s). - Returns a column that evaluates to a float, or an array of floats if the rank - argument is an array. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_quantile_float("sketch", sf.lit(0.5))).show() - +------------------------------------------+ - |kll_sketch_get_quantile_float(sketch, 0.5)| - +------------------------------------------+ - | 3.0| - +------------------------------------------+ + >>> df = spark.createDataFrame([(123,)], ['a']) + >>> df.select('*', sf.bitmap_bucket_number('a')).show() + +---+-----------------------+ + | a|bitmap_bucket_number(a)| + +---+-----------------------+ + |123| 1| + +---+-----------------------+ """ - fn = "kll_sketch_get_quantile_float" - return _invoke_function_over_columns(fn, sketch, rank) + return _invoke_function_over_columns("bitmap_bucket_number", col) @_try_remote_functions -def kll_sketch_get_quantile_double(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: +def bitmap_count(col: "ColumnOrName") -> Column: """ - Extracts a quantile value from a KLL double sketch given an input rank value. - The rank can be a single value or an array. + Returns the number of set bits in the input bitmap. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. - rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0). - A column that evaluates to a double or array. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + The input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - The quantile value(s). - Returns a column that evaluates to a double, or an array of doubles if the rank - argument is an array. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_or_agg` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_quantile_double("sketch", sf.lit(0.5))).show() - +-------------------------------------------+ - |kll_sketch_get_quantile_double(sketch, 0.5)| - +-------------------------------------------+ - | 3.0| - +-------------------------------------------+ + >>> df = spark.createDataFrame([("FFFF",)], ["a"]) + >>> df.select(sf.bitmap_count(sf.to_binary(df.a, sf.lit("hex")))).show() + +-------------------------------+ + |bitmap_count(to_binary(a, hex))| + +-------------------------------+ + | 16| + +-------------------------------+ """ - fn = "kll_sketch_get_quantile_double" - return _invoke_function_over_columns(fn, sketch, rank) + return _invoke_function_over_columns("bitmap_count", col) @_try_remote_functions -def kll_sketch_get_rank_bigint(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: +def bitmap_and(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Extracts a rank value from a KLL bigint sketch given an input quantile value. - The quantile can be a single value or an array. + Returns a bitmap that is the bitwise AND of two input bitmaps. - .. versionadded:: 4.1.0 + .. versionadded:: 4.4.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. - A column that evaluates to a binary. - quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup. - A column that evaluates to a long or array. Must be a constant. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - The rank value(s) (between 0.0 and 1.0). - Returns a column that evaluates to a double, or an array of doubles if the quantile - argument is an array. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_andnot` + :meth:`pyspark.sql.functions.bitmap_or` + :meth:`pyspark.sql.functions.bitmap_xor` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_rank_bigint("sketch", sf.lit(3))).show() - +-------------------------------------+ - |kll_sketch_get_rank_bigint(sketch, 3)| - +-------------------------------------+ - | 0.6| - +-------------------------------------+ + >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) + >>> df.select(sf.bitmap_and( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[70 00 00 00 00 0...| + +--------------------+ """ - fn = "kll_sketch_get_rank_bigint" - return _invoke_function_over_columns(fn, sketch, quantile) + return _invoke_function_over_columns("bitmap_and", left, right) @_try_remote_functions -def kll_sketch_get_rank_float(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: +def bitmap_or(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Extracts a rank value from a KLL float sketch given an input quantile value. - The quantile can be a single value or an array. + Returns a bitmap that is the bitwise OR of two input bitmaps. - .. versionadded:: 4.1.0 + .. versionadded:: 4.4.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. - quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup. - A column that evaluates to a float or array. Must be a constant. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - The rank value(s) (between 0.0 and 1.0). - Returns a column that evaluates to a double, or an array of doubles if the quantile - argument is an array. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_and` + :meth:`pyspark.sql.functions.bitmap_andnot` + :meth:`pyspark.sql.functions.bitmap_xor` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_rank_float("sketch", sf.lit(3.0))).show() - +--------------------------------------+ - |kll_sketch_get_rank_float(sketch, 3.0)| - +--------------------------------------+ - | 0.6| - +--------------------------------------+ + >>> df = spark.createDataFrame([("10", "20")], ["left", "right"]) + >>> df.select(sf.bitmap_or( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[30 00 00 00 00 0...| + +--------------------+ """ - fn = "kll_sketch_get_rank_float" - return _invoke_function_over_columns(fn, sketch, quantile) + return _invoke_function_over_columns("bitmap_or", left, right) @_try_remote_functions -def kll_sketch_get_rank_double(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: +def bitmap_andnot(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Extracts a rank value from a KLL double sketch given an input quantile value. - The quantile can be a single value or an array. + Returns a bitmap that is the bitwise AND NOT of two input bitmaps. - .. versionadded:: 4.1.0 + .. versionadded:: 4.4.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. - quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup. - A column that evaluates to a double or array. Must be a constant. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - The rank value(s) (between 0.0 and 1.0). - Returns a column that evaluates to a double, or an array of doubles if the quantile - argument is an array. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_and` + :meth:`pyspark.sql.functions.bitmap_or` + :meth:`pyspark.sql.functions.bitmap_xor` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_rank_double("sketch", sf.lit(3.0))).show() - +---------------------------------------+ - |kll_sketch_get_rank_double(sketch, 3.0)| - +---------------------------------------+ - | 0.6| - +---------------------------------------+ + >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) + >>> df.select(sf.bitmap_andnot( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[80 00 00 00 00 0...| + +--------------------+ """ - fn = "kll_sketch_get_rank_double" - return _invoke_function_over_columns(fn, sketch, quantile) + return _invoke_function_over_columns("bitmap_andnot", left, right) @_try_remote_functions -def theta_sketch_estimate(col: "ColumnOrName") -> Column: +def bitmap_xor(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns the estimated number of unique values given the binary representation - of a Datasketches ThetaSketch. + Returns a bitmap that is the bitwise XOR of two input bitmaps. - .. versionadded:: 4.1.0 + .. versionadded:: 4.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - - Returns - ------- - :class:`~pyspark.sql.Column` - The estimated number of unique values for the ThetaSketch. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. See Also -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_intersection` - :meth:`pyspark.sql.functions.theta_difference` - :meth:`pyspark.sql.functions.theta_union_agg` - :meth:`pyspark.sql.functions.theta_intersection_agg` - :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.bitmap_and` + :meth:`pyspark.sql.functions.bitmap_andnot` + :meth:`pyspark.sql.functions.bitmap_or` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_sketch_agg(value, 12))| - +--------------------------------------------------+ - | 3| - +--------------------------------------------------+ + >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) + >>> df.select(sf.bitmap_xor( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[80 00 00 00 00 0...| + +--------------------+ """ + return _invoke_function_over_columns("bitmap_xor", left, right) - fn = "theta_sketch_estimate" - return _invoke_function_over_columns(fn, col) + +# ---------------------- Datasketch Functions ---------------------- @_try_remote_functions -def theta_union( - col1: "ColumnOrName", col2: "ColumnOrName", lgNomEntries: Optional[Union[int, Column]] = None -) -> Column: +def hll_sketch_estimate(col: "ColumnOrName") -> Column: """ - Merges two binary representations of Datasketches ThetaSketch objects, using a - Datasketches Union object. + Returns the estimated number of unique values given the binary representation + of a Datasketches HllSketch. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - col2 : :class:`~pyspark.sql.Column` or column name - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries for the union operation - (must be between 4 and 26, defaults to 12) + col : :class:`~pyspark.sql.Column` or column name Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged ThetaSketch. + The estimated number of unique values for the HllSketch. See Also -------- - :meth:`pyspark.sql.functions.theta_union_agg` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + :meth:`pyspark.sql.functions.hll_union` + :meth:`pyspark.sql.functions.hll_union_agg` + :meth:`pyspark.sql.functions.hll_sketch_agg` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,4),(2,5),(2,5),(3,6)], "struct") - >>> df = df.agg( - ... sf.theta_sketch_agg("v1").alias("sketch1"), - ... sf.theta_sketch_agg("v2").alias("sketch2") - ... ) - >>> df.select(sf.theta_sketch_estimate(sf.theta_union(df.sketch1, "sketch2"))).show() - +--------------------------------------------------------+ - |theta_sketch_estimate(theta_union(sketch1, sketch2, 12))| - +--------------------------------------------------------+ - | 6| - +--------------------------------------------------------+ + >>> df = spark.createDataFrame([1,2,2,3], "INT") + >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() + +----------------------------------------------+ + |hll_sketch_estimate(hll_sketch_agg(value, 12))| + +----------------------------------------------+ + | 3| + +----------------------------------------------+ """ + from pyspark.sql.classic.column import _to_java_column - fn = "theta_union" - if lgNomEntries is not None: - return _invoke_function_over_columns( - fn, - col1, - col2, - lit(lgNomEntries), - ) - else: - return _invoke_function_over_columns(fn, col1, col2) + return _invoke_function("hll_sketch_estimate", _to_java_column(col)) @_try_remote_functions -def theta_intersection(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def hll_union( + col1: "ColumnOrName", col2: "ColumnOrName", allowDifferentLgConfigK: Optional[bool] = None +) -> Column: """ - Returns the intersection of two binary representations of Datasketches ThetaSketch - objects, using a Datasketches Intersection object. + Merges two binary representations of Datasketches HllSketch objects, using a + Datasketches Union object. Throws an exception if sketches have different + lgConfigK values and allowDifferentLgConfigK is unset or set to false. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name + allowDifferentLgConfigK : bool, optional + Allow sketches with different lgConfigK values to be merged (defaults to false). Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected ThetaSketch. + The binary representation of the merged HllSketch. See Also -------- - :meth:`pyspark.sql.functions.theta_intersection_agg` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + :meth:`pyspark.sql.functions.hll_union_agg` + :meth:`pyspark.sql.functions.hll_sketch_agg` + :meth:`pyspark.sql.functions.hll_sketch_estimate` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,1),(2,2),(3,2),(3,3)], "struct") + >>> df = spark.createDataFrame([(1,4),(2,5),(2,5),(3,6)], "struct") >>> df = df.agg( - ... sf.theta_sketch_agg("v1").alias("sketch1"), - ... sf.theta_sketch_agg("v2").alias("sketch2") + ... sf.hll_sketch_agg("v1").alias("sketch1"), + ... sf.hll_sketch_agg("v2").alias("sketch2") ... ) - >>> df.select(sf.theta_sketch_estimate(sf.theta_intersection(df.sketch1, "sketch2"))).show() - +-----------------------------------------------------------+ - |theta_sketch_estimate(theta_intersection(sketch1, sketch2))| - +-----------------------------------------------------------+ - | 3| - +-----------------------------------------------------------+ + >>> df.select(sf.hll_sketch_estimate(sf.hll_union(df.sketch1, "sketch2"))).show() + +-------------------------------------------------------+ + |hll_sketch_estimate(hll_union(sketch1, sketch2, false))| + +-------------------------------------------------------+ + | 6| + +-------------------------------------------------------+ """ + from pyspark.sql.classic.column import _to_java_column - fn = "theta_intersection" - return _invoke_function_over_columns(fn, col1, col2) + if allowDifferentLgConfigK is not None: + return _invoke_function( + "hll_union", + _to_java_column(col1), + _to_java_column(col2), + _enum_to_value(allowDifferentLgConfigK), + ) + else: + return _invoke_function("hll_union", _to_java_column(col1), _to_java_column(col2)) @_try_remote_functions -def theta_difference(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def kll_sketch_to_string_bigint(col: "ColumnOrName") -> Column: """ - Returns the set difference of two binary representations of Datasketches ThetaSketch - objects (elements in first sketch but not in second), using a Datasketches ANotB object. + Returns a string with human readable summary information about the KLL bigint sketch. .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - col2 : :class:`~pyspark.sql.Column` or column name + col : :class:`~pyspark.sql.Column` or column name + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the difference ThetaSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_intersection` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,4),(2,4),(3,5),(4,5)], "struct") - >>> df = df.agg( - ... sf.theta_sketch_agg("v1").alias("sketch1"), - ... sf.theta_sketch_agg("v2").alias("sketch2") - ... ) - >>> df.select(sf.theta_sketch_estimate(sf.theta_difference(df.sketch1, "sketch2"))).show() - +---------------------------------------------------------+ - |theta_sketch_estimate(theta_difference(sketch1, sketch2))| - +---------------------------------------------------------+ - | 3| - +---------------------------------------------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_to_string_bigint("sketch")).first()[0] + >>> "kll" in result.lower() + True """ - - fn = "theta_difference" - return _invoke_function_over_columns(fn, col1, col2) + fn = "kll_sketch_to_string_bigint" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def tuple_sketch_estimate_double(col: "ColumnOrName") -> Column: +def kll_sketch_to_string_float(col: "ColumnOrName") -> Column: """ - Returns the estimated number of distinct keys from a Datasketches TupleSketch - with double summaries. + Returns a string with human readable summary information about the KLL float sketch. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The estimated cardinality. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_sketch_summary_double` + A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +--------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| - +--------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_to_string_float("sketch")).first()[0] + >>> "kll" in result.lower() + True """ - fn = "tuple_sketch_estimate_double" + fn = "kll_sketch_to_string_float" return _invoke_function_over_columns(fn, col) @_try_remote_functions -def tuple_sketch_estimate_integer(col: "ColumnOrName") -> Column: +def kll_sketch_to_string_double(col: "ColumnOrName") -> Column: """ - Returns the estimated number of distinct keys from a Datasketches TupleSketch - with integer summaries. + Returns a string with human readable summary information about the KLL double sketch. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The estimated cardinality. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` + A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +----------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| - +----------------------------------------------------------------------------+ - | 2.0| - +----------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_to_string_double("sketch")).first()[0] + >>> "kll" in result.lower() + True """ - fn = "tuple_sketch_estimate_integer" + fn = "kll_sketch_to_string_double" return _invoke_function_over_columns(fn, col) @_try_remote_functions -def tuple_sketch_summary_double( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def kll_sketch_get_n_bigint(col: "ColumnOrName") -> Column: """ - Returns the aggregated summary value from a Datasketches TupleSketch with double summaries. + Returns the number of items collected in the KLL bigint sketch. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + The KLL bigint sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The aggregated summary value. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` + The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_summary_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +------------------------------------------------------------------------------+ - |tuple_sketch_summary_double(tuple_sketch_agg_double(key, value, 12, sum), sum)| - +------------------------------------------------------------------------------+ - | 60.0| - +------------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_n_bigint("sketch")).show() + +-------------------------------+ + |kll_sketch_get_n_bigint(sketch)| + +-------------------------------+ + | 5| + +-------------------------------+ """ - fn = "tuple_sketch_summary_double" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + fn = "kll_sketch_get_n_bigint" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def tuple_sketch_summary_integer( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def kll_sketch_get_n_float(col: "ColumnOrName") -> Column: """ - Returns the aggregated summary value from a Datasketches TupleSketch with integer summaries. + Returns the number of items collected in the KLL float sketch. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + The KLL float sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The aggregated summary value. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` + The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_summary_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +--------------------------------------------------------------------------------+ - |tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, value, 12, sum), sum)| - +--------------------------------------------------------------------------------+ - | 60| - +--------------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_n_float("sketch")).show() + +------------------------------+ + |kll_sketch_get_n_float(sketch)| + +------------------------------+ + | 5| + +------------------------------+ """ - fn = "tuple_sketch_summary_integer" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + fn = "kll_sketch_get_n_float" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def tuple_sketch_theta_double(col: "ColumnOrName") -> Column: +def kll_sketch_get_n_double(col: "ColumnOrName") -> Column: """ - Returns the theta value from a Datasketches TupleSketch with double summaries. + Returns the number of items collected in the KLL double sketch. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation + The KLL double sketch binary representation. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The theta value (between 0.0 and 1.0). - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` + The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_theta_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +-----------------------------------------------------------------------+ - |tuple_sketch_theta_double(tuple_sketch_agg_double(key, value, 12, sum))| - +-----------------------------------------------------------------------+ - | 1.0| - +-----------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_n_double("sketch")).show() + +-------------------------------+ + |kll_sketch_get_n_double(sketch)| + +-------------------------------+ + | 5| + +-------------------------------+ """ - return _invoke_function_over_columns("tuple_sketch_theta_double", col) + fn = "kll_sketch_get_n_double" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def tuple_sketch_theta_integer(col: "ColumnOrName") -> Column: +def kll_sketch_merge_bigint(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns the theta value from a Datasketches TupleSketch with integer summaries. + Merges two KLL bigint sketch buffers together into one. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation + left : :class:`~pyspark.sql.Column` or column name + The first KLL bigint sketch. + A column that evaluates to a binary. + right : :class:`~pyspark.sql.Column` or column name + The second KLL bigint sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The theta value (between 0.0 and 1.0). - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` + The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_theta_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +-------------------------------------------------------------------------+ - |tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, value, 12, sum))| - +-------------------------------------------------------------------------+ - | 1.0| - +-------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_merge_bigint("sketch", "sketch")).first()[0] + >>> result is not None and len(result) > 0 + True """ - return _invoke_function_over_columns("tuple_sketch_theta_integer", col) + fn = "kll_sketch_merge_bigint" + return _invoke_function_over_columns(fn, left, right) @_try_remote_functions -def tuple_union_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def kll_sketch_merge_float(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns the union of two Datasketches TupleSketch objects with double summaries. + Merges two KLL float sketch buffers together into one. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + left : :class:`~pyspark.sql.Column` or column name + The first KLL float sketch. + A column that evaluates to a binary. + right : :class:`~pyspark.sql.Column` or column name + The second KLL float sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_agg_double` - :meth:`pyspark.sql.functions.tuple_intersection_double` + The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 3, 30.0), (2, 20.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_double(df.sketch1, "sketch2"))).show() # noqa - +---------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_union_double(sketch1, sketch2, 12, sum))| - +---------------------------------------------------------------------------+ - | 4.0| - +---------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_merge_float("sketch", "sketch")).first()[0] + >>> result is not None and len(result) > 0 + True """ - fn = "tuple_union_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) + fn = "kll_sketch_merge_float" + return _invoke_function_over_columns(fn, left, right) @_try_remote_functions -def tuple_union_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def kll_sketch_merge_double(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns the union of two Datasketches TupleSketch objects with integer summaries. + Merges two KLL double sketch buffers together into one. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + left : :class:`~pyspark.sql.Column` or column name + The first KLL double sketch. + A column that evaluates to a binary. + right : :class:`~pyspark.sql.Column` or column name + The second KLL double sketch. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_agg_integer` - :meth:`pyspark.sql.functions.tuple_intersection_integer` + The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 3, 30), (2, 20, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_integer(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_union_integer(sketch1, sketch2, 12, sum))| - +-----------------------------------------------------------------------------+ - | 4.0| - +-----------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_merge_double("sketch", "sketch")).first()[0] + >>> result is not None and len(result) > 0 + True """ - fn = "tuple_union_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) + fn = "kll_sketch_merge_double" + return _invoke_function_over_columns(fn, left, right) @_try_remote_functions -def tuple_intersection_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def kll_sketch_get_quantile_bigint(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: """ - Returns the intersection of two Datasketches TupleSketch objects with double summaries. + Extracts a quantile value from a KLL bigint sketch given an input rank value. + The rank can be a single value or an array. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + sketch : :class:`~pyspark.sql.Column` or column name + The KLL bigint sketch binary representation. + A column that evaluates to a binary. + rank : :class:`~pyspark.sql.Column` or column name + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_double` - :meth:`pyspark.sql.functions.tuple_intersection_agg_double` + The quantile value(s). + Returns a column that evaluates to a long, or an array of longs if the rank + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 2, 20.0), (2, 20.0, 3, 30.0), (3, 30.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_double(df.sketch1, "sketch2"))).show() # noqa - +------------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_intersection_double(sketch1, sketch2, sum))| - +------------------------------------------------------------------------------+ - | 2.0| - +------------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_quantile_bigint("sketch", sf.lit(0.5))).show() + +-------------------------------------------+ + |kll_sketch_get_quantile_bigint(sketch, 0.5)| + +-------------------------------------------+ + | 3| + +-------------------------------------------+ """ - fn = "tuple_intersection_double" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) - else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + fn = "kll_sketch_get_quantile_bigint" + return _invoke_function_over_columns(fn, sketch, rank) @_try_remote_functions -def tuple_intersection_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def kll_sketch_get_quantile_float(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: """ - Returns the intersection of two Datasketches TupleSketch objects with integer summaries. + Extracts a quantile value from a KLL float sketch given an input rank value. + The rank can be a single value or an array. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + sketch : :class:`~pyspark.sql.Column` or column name + The KLL float sketch binary representation. + A column that evaluates to a binary. + rank : :class:`~pyspark.sql.Column` or column name + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_integer` - :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` + The quantile value(s). + Returns a column that evaluates to a float, or an array of floats if the rank + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 2, 20), (2, 20, 3, 30), (3, 30, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_integer(df.sketch1, "sketch2"))).show() # noqa - +--------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_intersection_integer(sketch1, sketch2, sum))| - +--------------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_quantile_float("sketch", sf.lit(0.5))).show() + +------------------------------------------+ + |kll_sketch_get_quantile_float(sketch, 0.5)| + +------------------------------------------+ + | 3.0| + +------------------------------------------+ """ - fn = "tuple_intersection_integer" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) - else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + fn = "kll_sketch_get_quantile_float" + return _invoke_function_over_columns(fn, sketch, rank) @_try_remote_functions -def tuple_difference_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def kll_sketch_get_quantile_double(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: """ - Returns the set difference of two Datasketches TupleSketch objects with double summaries - (elements in first sketch but not in second). + Extracts a quantile value from a KLL double sketch given an input rank value. + The rank can be a single value or an array. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column + sketch : :class:`~pyspark.sql.Column` or column name + The KLL double sketch binary representation. + A column that evaluates to a binary. + rank : :class:`~pyspark.sql.Column` or column name + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_double` - :meth:`pyspark.sql.functions.tuple_intersection_double` + The quantile value(s). + Returns a column that evaluates to a double, or an array of doubles if the rank + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 4, 40.0), (2, 20.0, 4, 40.0), (3, 30.0, 5, 50.0), (4, 40.0, 5, 50.0)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_double(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_difference_double(sketch1, sketch2))| - +-----------------------------------------------------------------------+ - | 3.0| - +-----------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_quantile_double("sketch", sf.lit(0.5))).show() + +-------------------------------------------+ + |kll_sketch_get_quantile_double(sketch, 0.5)| + +-------------------------------------------+ + | 3.0| + +-------------------------------------------+ """ - return _invoke_function_over_columns("tuple_difference_double", col1, col2) + fn = "kll_sketch_get_quantile_double" + return _invoke_function_over_columns(fn, sketch, rank) @_try_remote_functions -def tuple_difference_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def kll_sketch_get_rank_bigint(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: """ - Returns the set difference of two Datasketches TupleSketch objects with integer summaries - (elements in first sketch but not in second). + Extracts a rank value from a KLL bigint sketch given an input quantile value. + The quantile can be a single value or an array. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column + sketch : :class:`~pyspark.sql.Column` or column name + The KLL bigint sketch binary representation. + A column that evaluates to a binary. + quantile : :class:`~pyspark.sql.Column` or column name + The quantile value(s) to lookup. + A column that evaluates to a long or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_integer` - :meth:`pyspark.sql.functions.tuple_intersection_integer` + The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double, or an array of doubles if the quantile + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 4, 40), (2, 20, 4, 40), (3, 30, 5, 50), (4, 40, 5, 50)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_integer(df.sketch1, "sketch2"))).show() # noqa - +-------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_difference_integer(sketch1, sketch2))| - +-------------------------------------------------------------------------+ - | 3.0| - +-------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_rank_bigint("sketch", sf.lit(3))).show() + +-------------------------------------+ + |kll_sketch_get_rank_bigint(sketch, 3)| + +-------------------------------------+ + | 0.6| + +-------------------------------------+ """ - return _invoke_function_over_columns("tuple_difference_integer", col1, col2) + fn = "kll_sketch_get_rank_bigint" + return _invoke_function_over_columns(fn, sketch, quantile) @_try_remote_functions -def tuple_difference_theta_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def kll_sketch_get_rank_float(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: """ - Subtracts a Datasketches ThetaSketch from a TupleSketch with double summaries - (elements in TupleSketch but not in ThetaSketch). + Extracts a rank value from a KLL float sketch given an input quantile value. + The quantile can be a single value or an array. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with double summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column + sketch : :class:`~pyspark.sql.Column` or column name + The KLL float sketch binary representation. + A column that evaluates to a binary. + quantile : :class:`~pyspark.sql.Column` or column name + The quantile value(s) to lookup. + A column that evaluates to a float or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_double` - :meth:`pyspark.sql.functions.tuple_intersection_theta_double` + The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double, or an array of doubles if the quantile + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(5, 5.0, 4), (1, 1.0, 4), (2, 2.0, 5), (3, 3.0, 1)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_theta_double(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_difference_theta_double(sketch1, sketch2))| - +-----------------------------------------------------------------------------+ - | 2.0| - +-----------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_rank_float("sketch", sf.lit(3.0))).show() + +--------------------------------------+ + |kll_sketch_get_rank_float(sketch, 3.0)| + +--------------------------------------+ + | 0.6| + +--------------------------------------+ """ - return _invoke_function_over_columns("tuple_difference_theta_double", col1, col2) + fn = "kll_sketch_get_rank_float" + return _invoke_function_over_columns(fn, sketch, quantile) @_try_remote_functions -def tuple_difference_theta_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def kll_sketch_get_rank_double(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: """ - Subtracts a Datasketches ThetaSketch from a TupleSketch with integer summaries - (elements in TupleSketch but not in ThetaSketch). + Extracts a rank value from a KLL double sketch given an input quantile value. + The quantile can be a single value or an array. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with integer summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column + sketch : :class:`~pyspark.sql.Column` or column name + The KLL double sketch binary representation. + A column that evaluates to a binary. + quantile : :class:`~pyspark.sql.Column` or column name + The quantile value(s) to lookup. + A column that evaluates to a double or array. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_integer` - :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(5, 5, 4), (1, 1, 4), (2, 2, 5), (3, 3, 1)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_theta_integer(df.sketch1, "sketch2"))).show() # noqa - +-------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_difference_theta_integer(sketch1, sketch2))| - +-------------------------------------------------------------------------------+ - | 2.0| - +-------------------------------------------------------------------------------+ + The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double, or an array of doubles if the quantile + argument is an array. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_rank_double("sketch", sf.lit(3.0))).show() + +---------------------------------------+ + |kll_sketch_get_rank_double(sketch, 3.0)| + +---------------------------------------+ + | 0.6| + +---------------------------------------+ """ - return _invoke_function_over_columns("tuple_difference_theta_integer", col1, col2) + fn = "kll_sketch_get_rank_double" + return _invoke_function_over_columns(fn, sketch, quantile) @_try_remote_functions -def tuple_intersection_theta_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def theta_sketch_estimate(col: "ColumnOrName") -> Column: """ - Intersects a Datasketches TupleSketch with double summaries with a ThetaSketch. + Returns the estimated number of unique values given the binary representation + of a Datasketches ThetaSketch. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with double summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + col : :class:`~pyspark.sql.Column` or column name Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + The estimated number of unique values for the ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.theta_difference` + :meth:`pyspark.sql.functions.theta_union_agg` + :meth:`pyspark.sql.functions.theta_intersection_agg` :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_double` - :meth:`pyspark.sql.functions.tuple_intersection_agg_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1.0, 1), (2, 2.0, 2), (3, 3.0, 4)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_theta_double(df.sketch1, "sketch2"))).show() # noqa - +------------------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_intersection_theta_double(sketch1, sketch2, sum))| - +------------------------------------------------------------------------------------+ - | 2.0| - +------------------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([1,2,2,3], "INT") + >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_sketch_agg(value, 12))| + +--------------------------------------------------+ + | 3| + +--------------------------------------------------+ """ - fn = "tuple_intersection_theta_double" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) - else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + + fn = "theta_sketch_estimate" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def tuple_intersection_theta_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, +def theta_union( + col1: "ColumnOrName", col2: "ColumnOrName", lgNomEntries: Optional[Union[int, Column]] = None ) -> Column: """ - Intersects a Datasketches TupleSketch with integer summaries with a ThetaSketch. + Merges two binary representations of Datasketches ThetaSketch objects, using a + Datasketches Union object. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with integer summaries col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries for the union operation + (must be between 4 and 26, defaults to 12) Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + The binary representation of the merged ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.theta_union_agg` :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_integer` - :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1, 1), (2, 2, 2), (3, 3, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = spark.createDataFrame([(1,4),(2,5),(2,5),(3,6)], "struct") >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") + ... sf.theta_sketch_agg("v1").alias("sketch1"), + ... sf.theta_sketch_agg("v2").alias("sketch2") ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_theta_integer(df.sketch1, "sketch2"))).show() # noqa - +--------------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_intersection_theta_integer(sketch1, sketch2, sum))| - +--------------------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------------------+ + >>> df.select(sf.theta_sketch_estimate(sf.theta_union(df.sketch1, "sketch2"))).show() + +--------------------------------------------------------+ + |theta_sketch_estimate(theta_union(sketch1, sketch2, 12))| + +--------------------------------------------------------+ + | 6| + +--------------------------------------------------------+ """ - fn = "tuple_intersection_theta_integer" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) + + fn = "theta_union" + if lgNomEntries is not None: + return _invoke_function_over_columns( + fn, + col1, + col2, + lit(lgNomEntries), + ) else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + return _invoke_function_over_columns(fn, col1, col2) @_try_remote_functions -def tuple_union_theta_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def theta_intersection(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Merges a Datasketches TupleSketch with double summaries with a ThetaSketch. + Returns the intersection of two binary representations of Datasketches ThetaSketch + objects, using a Datasketches Intersection object. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with double summaries col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. + The binary representation of the intersected ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.theta_intersection_agg` :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_agg_double` - :meth:`pyspark.sql.functions.tuple_intersection_theta_double` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 3), (2, 20.0, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = spark.createDataFrame([(1,1),(2,2),(3,2),(3,3)], "struct") >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") + ... sf.theta_sketch_agg("v1").alias("sketch1"), + ... sf.theta_sketch_agg("v2").alias("sketch2") ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_theta_double(df.sketch1, "sketch2"))).show() # noqa - +---------------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_union_theta_double(sketch1, sketch2, 12, sum))| - +---------------------------------------------------------------------------------+ - | 4.0| - +---------------------------------------------------------------------------------+ + >>> df.select(sf.theta_sketch_estimate(sf.theta_intersection(df.sketch1, "sketch2"))).show() + +-----------------------------------------------------------+ + |theta_sketch_estimate(theta_intersection(sketch1, sketch2))| + +-----------------------------------------------------------+ + | 3| + +-----------------------------------------------------------+ """ - fn = "tuple_union_theta_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) + fn = "theta_intersection" + return _invoke_function_over_columns(fn, col1, col2) @_try_remote_functions -def tuple_union_theta_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def theta_difference(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Merges a Datasketches TupleSketch with integer summaries with a ThetaSketch. + Returns the set difference of two binary representations of Datasketches ThetaSketch + objects (elements in first sketch but not in second), using a Datasketches ANotB object. - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with integer summaries col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. + The binary representation of the difference ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_intersection` :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_agg_integer` - :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 3), (2, 20, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = spark.createDataFrame([(1,4),(2,4),(3,5),(4,5)], "struct") >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") + ... sf.theta_sketch_agg("v1").alias("sketch1"), + ... sf.theta_sketch_agg("v2").alias("sketch2") ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_theta_integer(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_union_theta_integer(sketch1, sketch2, 12, sum))| - +-----------------------------------------------------------------------------------+ - | 4.0| - +-----------------------------------------------------------------------------------+ + >>> df.select(sf.theta_sketch_estimate(sf.theta_difference(df.sketch1, "sketch2"))).show() + +---------------------------------------------------------+ + |theta_sketch_estimate(theta_difference(sketch1, sketch2))| + +---------------------------------------------------------+ + | 3| + +---------------------------------------------------------+ """ - fn = "tuple_union_theta_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) - - -# ---------------------- Predicates functions ------------------------------ + fn = "theta_difference" + return _invoke_function_over_columns(fn, col1, col2) @_try_remote_functions -def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def tuple_sketch_estimate_double(col: "ColumnOrName") -> Column: """ - Returns `col2` if `col1` is null, or `col1` otherwise. + Returns the estimated number of distinct keys from a Datasketches TupleSketch + with double summaries. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - col2 : :class:`~pyspark.sql.Column` or str + col : :class:`~pyspark.sql.Column` or column name + The column containing a binary TupleSketch representation + + Returns + ------- + :class:`~pyspark.sql.Column` + The estimated cardinality. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_sketch_summary_double` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) - >>> df.select(sf.ifnull(df.e, sf.lit(8))).show() - +------------+ - |ifnull(e, 8)| - +------------+ - | 8| - | 1| - +------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +--------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| + +--------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("ifnull", col1, col2) + fn = "tuple_sketch_estimate_double" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def isnotnull(col: "ColumnOrName") -> Column: +def tuple_sketch_estimate_integer(col: "ColumnOrName") -> Column: """ - Returns true if `col` is not null, or false otherwise. + Returns the estimated number of distinct keys from a Datasketches TupleSketch + with integer summaries. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + The column containing a binary TupleSketch representation + + Returns + ------- + :class:`~pyspark.sql.Column` + The estimated cardinality. See Also -------- - :meth:`pyspark.sql.functions.isnan` - :meth:`pyspark.sql.functions.isnull` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) - >>> df.select('*', sf.isnotnull(df.e)).show() - +----+---------------+ - | e|(e IS NOT NULL)| - +----+---------------+ - |NULL| false| - | 1| true| - +----+---------------+ - - >>> df.select('*', sf.isnotnull('e')).show() - +----+---------------+ - | e|(e IS NOT NULL)| - +----+---------------+ - |NULL| false| - | 1| true| - +----+---------------+ + >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +----------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| + +----------------------------------------------------------------------------+ + | 2.0| + +----------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("isnotnull", col) + fn = "tuple_sketch_estimate_integer" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def tuple_sketch_summary_double( + col: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns same result as the EQUAL(=) operator for non-null operands, - but returns true if both are null, false if one of them is null. + Returns the aggregated summary value from a Datasketches TupleSketch with double summaries. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. - col2 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) - >>> df.select('*', sf.equal_null(df.a, df.b)).show() - +----+----+----------------+ - | a| b|equal_null(a, b)| - +----+----+----------------+ - |NULL|NULL| true| - | 1| 9| false| - +----+----+----------------+ - - >>> df.select('*', sf.equal_null('a', 'b')).show() - +----+----+----------------+ - | a| b|equal_null(a, b)| - +----+----+----------------+ - |NULL|NULL| true| - | 1| 9| false| - +----+----+----------------+ - """ - return _invoke_function_over_columns("equal_null", col1, col2) - - -@_try_remote_functions -def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """ - Returns null if `col1` equals to `col2`, or `col1` otherwise. + col : :class:`~pyspark.sql.Column` or column name + The column containing a binary TupleSketch representation + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" - .. versionadded:: 3.5.0 + Returns + ------- + :class:`~pyspark.sql.Column` + The aggregated summary value. - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. - col2 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) - >>> df.select('*', sf.nullif(df.a, df.b)).show() - +----+----+------------+ - | a| b|nullif(a, b)| - +----+----+------------+ - |NULL|NULL| NULL| - | 1| 9| 1| - +----+----+------------+ - - >>> df.select('*', sf.nullif('a', 'b')).show() - +----+----+------------+ - | a| b|nullif(a, b)| - +----+----+------------+ - |NULL|NULL| NULL| - | 1| 9| 1| - +----+----+------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_summary_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +------------------------------------------------------------------------------+ + |tuple_sketch_summary_double(tuple_sketch_agg_double(key, value, 12, sum), sum)| + +------------------------------------------------------------------------------+ + | 60.0| + +------------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("nullif", col1, col2) + fn = "tuple_sketch_summary_double" + if mode is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(mode)) @_try_remote_functions -def nullifzero(col: "ColumnOrName") -> Column: +def tuple_sketch_summary_integer( + col: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns null if `col` is equal to zero, or `col` otherwise. + Returns the aggregated summary value from a Datasketches TupleSketch with integer summaries. - .. versionadded:: 4.0.0 + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric. - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(0,), (1,)], ["a"]) - >>> df.select('*', sf.nullifzero(df.a)).show() - +---+-------------+ - | a|nullifzero(a)| - +---+-------------+ - | 0| NULL| - | 1| 1| - +---+-------------+ - - >>> df.select('*', sf.nullifzero('a')).show() - +---+-------------+ - | a|nullifzero(a)| - +---+-------------+ - | 0| NULL| - | 1| 1| - +---+-------------+ - """ - return _invoke_function_over_columns("nullifzero", col) - - -@_try_remote_functions -def nvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """ - Returns `col2` if `col1` is null, or `col1` otherwise. - - .. versionadded:: 3.5.0 + The column containing a binary TupleSketch representation + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - col2 : :class:`~pyspark.sql.Column` or column name + Returns + ------- + :class:`~pyspark.sql.Column` + The aggregated summary value. See Also -------- - :meth:`pyspark.sql.functions.nvl2` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None, 8,), (1, 9,)], ["a", "b"]) - >>> df.select('*', sf.nvl(df.a, df.b)).show() - +----+---+---------+ - | a| b|nvl(a, b)| - +----+---+---------+ - |NULL| 8| 8| - | 1| 9| 1| - +----+---+---------+ - - >>> df.select('*', sf.nvl('a', 'b')).show() - +----+---+---------+ - | a| b|nvl(a, b)| - +----+---+---------+ - |NULL| 8| 8| - | 1| 9| 1| - +----+---+---------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_summary_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +--------------------------------------------------------------------------------+ + |tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, value, 12, sum), sum)| + +--------------------------------------------------------------------------------+ + | 60| + +--------------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("nvl", col1, col2) + fn = "tuple_sketch_summary_integer" + if mode is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(mode)) @_try_remote_functions -def nvl2(col1: "ColumnOrName", col2: "ColumnOrName", col3: "ColumnOrName") -> Column: +def tuple_sketch_theta_double(col: "ColumnOrName") -> Column: """ - Returns `col2` if `col1` is not null, or `col3` otherwise. + Returns the theta value from a Datasketches TupleSketch with double summaries. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - col2 : :class:`~pyspark.sql.Column` or column name - col3 : :class:`~pyspark.sql.Column` or column name + col : :class:`~pyspark.sql.Column` or column name + The column containing a binary TupleSketch representation + + Returns + ------- + :class:`~pyspark.sql.Column` + The theta value (between 0.0 and 1.0). See Also -------- - :meth:`pyspark.sql.functions.nvl` + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None, 8, 6,), (1, 9, 9,)], ["a", "b", "c"]) - >>> df.select('*', sf.nvl2(df.a, df.b, df.c)).show() - +----+---+---+-------------+ - | a| b| c|nvl2(a, b, c)| - +----+---+---+-------------+ - |NULL| 8| 6| 6| - | 1| 9| 9| 9| - +----+---+---+-------------+ - - >>> df.select('*', sf.nvl2('a', 'b', 'c')).show() - +----+---+---+-------------+ - | a| b| c|nvl2(a, b, c)| - +----+---+---+-------------+ - |NULL| 8| 6| 6| - | 1| 9| 9| 9| - +----+---+---+-------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_theta_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +-----------------------------------------------------------------------+ + |tuple_sketch_theta_double(tuple_sketch_agg_double(key, value, 12, sum))| + +-----------------------------------------------------------------------+ + | 1.0| + +-----------------------------------------------------------------------+ """ - return _invoke_function_over_columns("nvl2", col1, col2, col3) + return _invoke_function_over_columns("tuple_sketch_theta_double", col) @_try_remote_functions -def zeroifnull(col: "ColumnOrName") -> Column: +def tuple_sketch_theta_integer(col: "ColumnOrName") -> Column: """ - Returns zero if `col` is null, or `col` otherwise. + Returns the theta value from a Datasketches TupleSketch with integer summaries. - .. versionadded:: 4.0.0 + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + The column containing a binary TupleSketch representation - Examples + Returns + ------- + :class:`~pyspark.sql.Column` + The theta value (between 0.0 and 1.0). + + See Also -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None,), (1,)], ["a"]) - >>> df.select('*', sf.zeroifnull(df.a)).show() - +----+-------------+ - | a|zeroifnull(a)| - +----+-------------+ - |NULL| 0| - | 1| 1| - +----+-------------+ + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` - >>> df.select('*', sf.zeroifnull('a')).show() - +----+-------------+ - | a|zeroifnull(a)| - +----+-------------+ - |NULL| 0| - | 1| 1| - +----+-------------+ + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_theta_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +-------------------------------------------------------------------------+ + |tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, value, 12, sum))| + +-------------------------------------------------------------------------+ + | 1.0| + +-------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("zeroifnull", col) + return _invoke_function_over_columns("tuple_sketch_theta_integer", col) @_try_remote_functions -def hmac( - key: "ColumnOrName", - message: "ColumnOrName", - algorithm: Optional["ColumnOrName"] = None, +def tuple_union_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, ) -> Column: """ - Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the - given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with :func:`hex` or - :func:`base64` for a textual value. The default algorithm is 'SHA-256'. + Returns the union of two Datasketches TupleSketch objects with double summaries. - .. versionadded:: 4.3.0 + .. versionadded:: 4.2.0 Parameters ---------- - key : :class:`~pyspark.sql.Column` or column name - The secret key, as a binary value. - message : :class:`~pyspark.sql.Column` or column name - The message to authenticate, as a binary value. - algorithm : :class:`~pyspark.sql.Column` or column name, optional - The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. - The default is SHA-256. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - A new column that contains the raw HMAC bytes. + The binary representation of the merged TupleSketch. - Examples + See Also -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_agg_double` + :meth:`pyspark.sql.functions.tuple_intersection_double` - Example 1: Compute the HMAC with the default SHA-256 algorithm. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) - >>> df.select(sf.hex(sf.hmac(df.key, df.message))).show(truncate=False) - +----------------------------------------------------------------+ - |hex(hmac(key, message, SHA-256)) | - +----------------------------------------------------------------+ - |6E9EF29B75FFFC5B7ABAE527D58FDADB2FE42E7219011976917343065F58ED4A| - +----------------------------------------------------------------+ - - Example 2: Compute the HMAC with an explicit algorithm. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) - >>> df.select(sf.hex(sf.hmac(df.key, df.message, sf.lit("SHA-1")))).show(truncate=False) - +----------------------------------------+ - |hex(hmac(key, message, SHA-1)) | - +----------------------------------------+ - |2088DF74D5F2146B48146CAF4965377E9D0BE3A4| - +----------------------------------------+ + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0, 3, 30.0), (2, 20.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_double(df.sketch1, "sketch2"))).show() # noqa + +---------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_union_double(sketch1, sketch2, 12, sum))| + +---------------------------------------------------------------------------+ + | 4.0| + +---------------------------------------------------------------------------+ """ - if algorithm is None: - return _invoke_function_over_columns("hmac", key, message) - else: - return _invoke_function_over_columns("hmac", key, message, algorithm) + fn = "tuple_union_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) @_try_remote_functions -def aes_encrypt( - input: "ColumnOrName", - key: "ColumnOrName", - mode: Optional["ColumnOrName"] = None, - padding: Optional["ColumnOrName"] = None, - iv: Optional["ColumnOrName"] = None, - aad: Optional["ColumnOrName"] = None, +def tuple_union_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, ) -> Column: """ - Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. - Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, - `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization - vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 - bytes for GCM. If not provided, a random vector will be generated and prepended to the - output. Optional additional authenticated data (AAD) is only supported for GCM. If provided - for encryption, the identical AAD value must be provided for decryption. The default mode is - GCM. + Returns the union of two Datasketches TupleSketch objects with integer summaries. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - input : :class:`~pyspark.sql.Column` or column name - The binary value to encrypt. - A column that evaluates to a binary. - key : :class:`~pyspark.sql.Column` or column name - The passphrase to use to encrypt the data. - A column that evaluates to a binary. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. mode : :class:`~pyspark.sql.Column` or str, optional - Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - GCM, CBC. - A column that evaluates to a string. - padding : :class:`~pyspark.sql.Column` or column name, optional - Specifies how to pad messages whose length is not a multiple of the block size. Valid - values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - for CBC. - A column that evaluates to a string. - iv : :class:`~pyspark.sql.Column` or column name, optional - Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - "". 16-byte array for CBC mode. 12-byte array for GCM mode. - A column that evaluates to a binary. - aad : :class:`~pyspark.sql.Column` or column name, optional - Optional additional authenticated data. Only supported for GCM mode. This can be any - free-form input and must be provided for both encryption and decryption. - A column that evaluates to a binary. + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an encrypted value. - Returns a column that evaluates to a binary. + The binary representation of the merged TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.aes_decrypt` - :meth:`pyspark.sql.functions.try_aes_decrypt` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_agg_integer` + :meth:`pyspark.sql.functions.tuple_intersection_integer` Examples -------- - - Example 1: Encrypt data with key, mode, padding, iv and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "000000000000000000000000", "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "iv", "aad"] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10, 3, 30), (2, 20, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") ... ) - >>> df.select(sf.base64(sf.aes_encrypt( - ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex")), df.aad) - ... )).show(truncate=False) - +-----------------------------------------------------------------------+ - |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), aad))| - +-----------------------------------------------------------------------+ - |AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4 | - +-----------------------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_integer(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_union_integer(sketch1, sketch2, 12, sum))| + +-----------------------------------------------------------------------------+ + | 4.0| + +-----------------------------------------------------------------------------+ + """ + fn = "tuple_union_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) - Example 2: Encrypt data with key, mode, padding and iv. + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "000000000000000000000000", "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "iv", "aad"] - ... ) - >>> df.select(sf.base64(sf.aes_encrypt( - ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex"))) - ... )).show(truncate=False) - +--------------------------------------------------------------------+ - |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), ))| - +--------------------------------------------------------------------+ - |AAAAAAAAAAAAAAAAQiYi+sRNYDAOTjdSEcYBFsAWPL1f | - +--------------------------------------------------------------------+ - Example 3: Encrypt data with key, mode and padding. +@_try_remote_functions +def tuple_intersection_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Returns the intersection of two Datasketches TupleSketch objects with double summaries. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark SQL", "1234567890abcdef", "ECB", "PKCS",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode", df.padding), - ... df.key, df.mode, df.padding - ... ).cast("STRING")).show(truncate=False) - +---------------------------------------------------------------------------------------------+ - |CAST(aes_decrypt(aes_encrypt(input, key, mode, padding, , ), key, mode, padding, ) AS STRING)| - +---------------------------------------------------------------------------------------------+ - |Spark SQL | - +---------------------------------------------------------------------------------------------+ + .. versionadded:: 4.2.0 - Example 4: Encrypt data with key and mode. + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark SQL", "0000111122223333", "ECB",)], - ... ["input", "key", "mode"] - ... ) - >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode"), - ... df.key, df.mode - ... ).cast("STRING")).show(truncate=False) - +---------------------------------------------------------------------------------------------+ - |CAST(aes_decrypt(aes_encrypt(input, key, mode, DEFAULT, , ), key, mode, DEFAULT, ) AS STRING)| - +---------------------------------------------------------------------------------------------+ - |Spark SQL | - +---------------------------------------------------------------------------------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the intersected TupleSketch. - Example 5: Encrypt data with key. + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_double` + :meth:`pyspark.sql.functions.tuple_intersection_agg_double` - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark SQL", "abcdefghijklmnop",)], - ... ["input", "key"] + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0, 2, 20.0), (2, 20.0, 3, 30.0), (3, 30.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(sf.base64(sf.aes_encrypt(df.input, df.key))), df.key - ... ).cast("STRING")).show(truncate=False) - +-------------------------------------------------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(base64(aes_encrypt(input, key, GCM, DEFAULT, , ))), key, GCM, DEFAULT, ) AS STRING)| - +-------------------------------------------------------------------------------------------------------------+ - |Spark SQL | - +-------------------------------------------------------------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_double(df.sketch1, "sketch2"))).show() # noqa + +------------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_intersection_double(sketch1, sketch2, sum))| + +------------------------------------------------------------------------------+ + | 2.0| + +------------------------------------------------------------------------------+ """ - _mode = lit("GCM") if mode is None else mode - _padding = lit("DEFAULT") if padding is None else padding - _iv = lit("") if iv is None else iv - _aad = lit("") if aad is None else aad - return _invoke_function_over_columns("aes_encrypt", input, key, _mode, _padding, _iv, _aad) + fn = "tuple_intersection_double" + if mode is None: + return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) @_try_remote_functions -def aes_decrypt( - input: "ColumnOrName", - key: "ColumnOrName", - mode: Optional["ColumnOrName"] = None, - padding: Optional["ColumnOrName"] = None, - aad: Optional["ColumnOrName"] = None, +def tuple_intersection_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, ) -> Column: """ - Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, - 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', - 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is - only supported for GCM. If provided for encryption, the identical AAD value must be provided - for decryption. The default mode is GCM. + Returns the intersection of two Datasketches TupleSketch objects with integer summaries. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - input : :class:`~pyspark.sql.Column` or column name - The binary value to decrypt. - A column that evaluates to a binary. - key : :class:`~pyspark.sql.Column` or column name - The passphrase to use to decrypt the data. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or column name, optional - Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - GCM, CBC. - A column that evaluates to a string. - padding : :class:`~pyspark.sql.Column` or column name, optional - Specifies how to pad messages whose length is not a multiple of the block size. Valid - values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - for CBC. - A column that evaluates to a string. - aad : :class:`~pyspark.sql.Column` or column name, optional - Optional additional authenticated data. Only supported for GCM mode. This can be any - free-form input and must be provided for both encryption and decryption. - A column that evaluates to a binary. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a decrypted value. - Returns a column that evaluates to a binary. + The binary representation of the intersected TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.aes_encrypt` - :meth:`pyspark.sql.functions.try_aes_decrypt` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_integer` + :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` Examples -------- - - Example 1: Decrypt data with key, mode, padding and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", - ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "aad"] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10, 2, 20), (2, 20, 3, 30), (3, 30, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad - ... ).cast("STRING")).show(truncate=False) - +---------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| - +---------------------------------------------------------------------+ - |Spark | - +---------------------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_integer(df.sketch1, "sketch2"))).show() # noqa + +--------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_intersection_integer(sketch1, sketch2, sum))| + +--------------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------------+ + """ + fn = "tuple_intersection_integer" + if mode is None: + return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) - Example 2: Decrypt data with key, mode and padding. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding - ... ).cast("STRING")).show(truncate=False) - +------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| - +------------------------------------------------------------------+ - |Spark | - +------------------------------------------------------------------+ +@_try_remote_functions +def tuple_difference_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """ + Returns the set difference of two Datasketches TupleSketch objects with double summaries + (elements in first sketch but not in second). - Example 3: Decrypt data with key and mode. + .. versionadded:: 4.2.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode" - ... ).cast("STRING")).show(truncate=False) - +------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| - +------------------------------------------------------------------+ - |Spark | - +------------------------------------------------------------------+ + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column - Example 4: Decrypt data with key. + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the difference TupleSketch. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", - ... "0000111122223333",)], - ... ["input", "key"] + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_double` + :meth:`pyspark.sql.functions.tuple_intersection_double` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0, 4, 40.0), (2, 20.0, 4, 40.0), (3, 30.0, 5, 50.0), (4, 40.0, 5, 50.0)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unhex(df.input), df.key - ... ).cast("STRING")).show(truncate=False) - +--------------------------------------------------------------+ - |CAST(aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| - +--------------------------------------------------------------+ - |Spark | - +--------------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_double(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_difference_double(sketch1, sketch2))| + +-----------------------------------------------------------------------+ + | 3.0| + +-----------------------------------------------------------------------+ """ - _mode = lit("GCM") if mode is None else mode - _padding = lit("DEFAULT") if padding is None else padding - _aad = lit("") if aad is None else aad - return _invoke_function_over_columns("aes_decrypt", input, key, _mode, _padding, _aad) + return _invoke_function_over_columns("tuple_difference_double", col1, col2) @_try_remote_functions -def try_aes_decrypt( - input: "ColumnOrName", - key: "ColumnOrName", - mode: Optional["ColumnOrName"] = None, - padding: Optional["ColumnOrName"] = None, - aad: Optional["ColumnOrName"] = None, -) -> Column: +def tuple_difference_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - This is a special version of `aes_decrypt` that performs the same operation, - but returns a NULL value instead of raising an error if the decryption cannot be performed. - Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, - 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', - 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is - only supported for GCM. If provided for encryption, the identical AAD value must be provided - for decryption. The default mode is GCM. + Returns the set difference of two Datasketches TupleSketch objects with integer summaries + (elements in first sketch but not in second). - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - input : :class:`~pyspark.sql.Column` or column name - The binary value to decrypt. - A column that evaluates to a binary. - key : :class:`~pyspark.sql.Column` or column name - The passphrase to use to decrypt the data. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or column name, optional - Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - GCM, CBC. - A column that evaluates to a string. - padding : :class:`~pyspark.sql.Column` or column name, optional - Specifies how to pad messages whose length is not a multiple of the block size. Valid - values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - for CBC. - A column that evaluates to a string. - aad : :class:`~pyspark.sql.Column` or column name, optional - Optional additional authenticated data. Only supported for GCM mode. This can be any - free-form input and must be provided for both encryption and decryption. - A column that evaluates to a binary. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a decrypted value or a NULL value. - Returns a column that evaluates to a binary. + The binary representation of the difference TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.aes_encrypt` - :meth:`pyspark.sql.functions.aes_decrypt` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_integer` + :meth:`pyspark.sql.functions.tuple_intersection_integer` Examples -------- - - Example 1: Decrypt data with key, mode, padding and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", - ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "aad"] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10, 4, 40), (2, 20, 4, 40), (3, 30, 5, 50), (4, 40, 5, 50)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad - ... ).cast("STRING")).show(truncate=False) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_integer(df.sketch1, "sketch2"))).show() # noqa +-------------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| + |tuple_sketch_estimate_integer(tuple_difference_integer(sketch1, sketch2))| +-------------------------------------------------------------------------+ - |Spark | + | 3.0| +-------------------------------------------------------------------------+ + """ + return _invoke_function_over_columns("tuple_difference_integer", col1, col2) - Example 2: Failed to decrypt data with key, mode, padding and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT", - ... "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "aad"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad - ... ).cast("STRING")).show(truncate=False) - +-------------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| - +-------------------------------------------------------------------------+ - |NULL | - +-------------------------------------------------------------------------+ - Example 3: Decrypt data with key, mode and padding. +@_try_remote_functions +def tuple_difference_theta_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """ + Subtracts a Datasketches ThetaSketch from a TupleSketch with double summaries + (elements in TupleSketch but not in ThetaSketch). - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding - ... ).cast("STRING")).show(truncate=False) - +----------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| - +----------------------------------------------------------------------+ - |Spark | - +----------------------------------------------------------------------+ + .. versionadded:: 4.2.0 - Example 4: Decrypt data with key and mode. + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with double summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode" - ... ).cast("STRING")).show(truncate=False) - +----------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| - +----------------------------------------------------------------------+ - |Spark | - +----------------------------------------------------------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the difference TupleSketch. - Example 5: Decrypt data with key. + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_theta_double` + :meth:`pyspark.sql.functions.tuple_intersection_theta_double` - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", - ... "0000111122223333",)], - ... ["input", "key"] + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(5, 5.0, 4), (1, 1.0, 4), (2, 2.0, 5), (3, 3.0, 1)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unhex(df.input), df.key - ... ).cast("STRING")).show(truncate=False) - +------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| - +------------------------------------------------------------------+ - |Spark | - +------------------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_theta_double(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_difference_theta_double(sketch1, sketch2))| + +-----------------------------------------------------------------------------+ + | 2.0| + +-----------------------------------------------------------------------------+ """ - _mode = lit("GCM") if mode is None else mode - _padding = lit("DEFAULT") if padding is None else padding - _aad = lit("") if aad is None else aad - return _invoke_function_over_columns("try_aes_decrypt", input, key, _mode, _padding, _aad) + return _invoke_function_over_columns("tuple_difference_theta_double", col1, col2) @_try_remote_functions -def sha(col: "ColumnOrName") -> Column: +def tuple_difference_theta_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns a sha1 hash value as a hex string of the `col`. + Subtracts a Datasketches ThetaSketch from a TupleSketch with integer summaries + (elements in TupleSketch but not in ThetaSketch). - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a binary. + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with integer summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the difference TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.sha1` - :meth:`pyspark.sql.functions.sha2` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_theta_integer` + :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.sha(sf.lit("Spark"))).show() - +--------------------+ - | sha(Spark)| - +--------------------+ - |85f5955f4b27a9a4c...| - +--------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(5, 5, 4), (1, 1, 4), (2, 2, 5), (3, 3, 1)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_theta_integer(df.sketch1, "sketch2"))).show() # noqa + +-------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_difference_theta_integer(sketch1, sketch2))| + +-------------------------------------------------------------------------------+ + | 2.0| + +-------------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("sha", col) + return _invoke_function_over_columns("tuple_difference_theta_integer", col1, col2) @_try_remote_functions -def input_file_block_length() -> Column: +def tuple_intersection_theta_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns the length of the block being read, or -1 if not available. + Intersects a Datasketches TupleSketch with double summaries with a ThetaSketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 + + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with double summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the intersected TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.input_file_name` - :meth:`pyspark.sql.functions.input_file_block_start` + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_theta_double` + :meth:`pyspark.sql.functions.tuple_intersection_agg_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") - >>> df.select(sf.input_file_block_length()).show() - +-------------------------+ - |input_file_block_length()| - +-------------------------+ - | 87| - | 87| - | 87| - | 87| - | 87| - | 87| - | 87| - | 87| - +-------------------------+ + >>> df = spark.createDataFrame([(1, 1.0, 1), (2, 2.0, 2), (3, 3.0, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_theta_double(df.sketch1, "sketch2"))).show() # noqa + +------------------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_intersection_theta_double(sketch1, sketch2, sum))| + +------------------------------------------------------------------------------------+ + | 2.0| + +------------------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("input_file_block_length") + fn = "tuple_intersection_theta_double" + if mode is None: + return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) @_try_remote_functions -def input_file_block_start() -> Column: +def tuple_intersection_theta_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns the start offset of the block being read, or -1 if not available. + Intersects a Datasketches TupleSketch with integer summaries with a ThetaSketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 + + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with integer summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the intersected TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.input_file_name` - :meth:`pyspark.sql.functions.input_file_block_length` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_theta_integer` + :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") - >>> df.select(sf.input_file_block_start()).show() - +------------------------+ - |input_file_block_start()| - +------------------------+ - | 0| - | 0| - | 0| - | 0| - | 0| - | 0| - | 0| - | 0| - +------------------------+ + >>> df = spark.createDataFrame([(1, 1, 1), (2, 2, 2), (3, 3, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_theta_integer(df.sketch1, "sketch2"))).show() # noqa + +--------------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_intersection_theta_integer(sketch1, sketch2, sum))| + +--------------------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------------------+ """ - return _invoke_function_over_columns("input_file_block_start") + fn = "tuple_intersection_theta_integer" + if mode is None: + return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) @_try_remote_functions -def reflect(*cols: "ColumnOrName") -> Column: +def tuple_union_theta_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Calls a method with reflection. + Merges a Datasketches TupleSketch with double summaries with a ThetaSketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a Column representing literal string for the class name, - and the second element should be a Column representing literal string for the method name, - and the remaining are input arguments (Columns or column names) to the Java method. + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with double summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the merged TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.java_method` - :meth:`pyspark.sql.functions.try_reflect` + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_agg_double` + :meth:`pyspark.sql.functions.tuple_intersection_theta_double` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) - >>> df.select( - ... sf.reflect(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') - ... ).show(truncate=False) - +--------------------------------------+ - |reflect(java.util.UUID, fromString, a)| - +--------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +--------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0, 3), (2, 20.0, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_theta_double(df.sketch1, "sketch2"))).show() # noqa + +---------------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_union_theta_double(sketch1, sketch2, 12, sum))| + +---------------------------------------------------------------------------------+ + | 4.0| + +---------------------------------------------------------------------------------+ """ - return _invoke_function_over_seq_of_columns("reflect", cols) + fn = "tuple_union_theta_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) @_try_remote_functions -def java_method(*cols: "ColumnOrName") -> Column: +def tuple_union_theta_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Calls a method with reflection. + Merges a Datasketches TupleSketch with integer summaries with a ThetaSketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a Column representing literal string for the class name, - and the second element should be a Column representing literal string for the method name, - and the remaining are input arguments (Columns or column names) to the Java method. + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with integer summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the merged TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.reflect` - :meth:`pyspark.sql.functions.try_reflect` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_agg_integer` + :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` Examples -------- - Example 1: Reflecting a method call with a column argument + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10, 3), (2, 20, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_theta_integer(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_union_theta_integer(sketch1, sketch2, 12, sum))| + +-----------------------------------------------------------------------------------+ + | 4.0| + +-----------------------------------------------------------------------------------+ + """ + fn = "tuple_union_theta_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select( - ... sf.java_method( - ... sf.lit("java.util.UUID"), - ... sf.lit("fromString"), - ... sf.lit("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2") - ... ) - ... ).show(truncate=False) - +-----------------------------------------------------------------------------+ - |java_method(java.util.UUID, fromString, a5cf6c42-0c85-418f-af6c-3e4e5b1328f2)| - +-----------------------------------------------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +-----------------------------------------------------------------------------+ + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) - Example 2: Reflecting a method call with a column name argument - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) - >>> df.select( - ... sf.java_method(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') - ... ).show(truncate=False) - +------------------------------------------+ - |java_method(java.util.UUID, fromString, a)| - +------------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +------------------------------------------+ - """ - return _invoke_function_over_seq_of_columns("java_method", cols) +# ---------------------- Geospatial ST Functions ---------------------- + + +def _ensure_column_or_name(arg: Optional[Any]) -> "ColumnOrName": + if not isinstance(arg, (Column, str)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "arg", + "arg_type": type(arg).__name__, + }, + ) + return arg @_try_remote_functions -def try_reflect(*cols: "ColumnOrName") -> Column: - """ - This is a special version of `reflect` that performs the same operation, but returns a NULL - value instead of raising an error if the invoke method thrown exception. +def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None) -> Column: + """Returns the input GEOGRAPHY or GEOMETRY value in WKB format. + .. versionadded:: 4.1.0 - .. versionadded:: 4.0.0 + .. versionchanged:: 4.2.0 + Added the optional `endianness` parameter. Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a Column representing literal string for the class name, - and the second element should be a Column representing literal string for the method name, - and the remaining are input arguments (Columns or column names) to the Java method. - - See Also - -------- - :meth:`pyspark.sql.functions.reflect` - :meth:`pyspark.sql.functions.java_method` + geo : :class:`~pyspark.sql.Column` or str + A geospatial value, either a GEOGRAPHY or a GEOMETRY. + endianness : :class:`~pyspark.sql.Column` or str, optional + The optional endianness of the output WKB, 'NDR' for little-endian (default) or 'XDR' for + big-endian. Examples -------- - Example 1: Reflecting a method call with arguments + Example 1: Getting WKB from GEOGRAPHY. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2",)], ["a"]) - >>> df.select( - ... sf.try_reflect(sf.lit("java.util.UUID"), sf.lit("fromString"), "a") - ... ).show(truncate=False) - +------------------------------------------+ - |try_reflect(java.util.UUID, fromString, a)| - +------------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +------------------------------------------+ + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] - Example 2: Exception in the reflection call, resulting in null + Example 2: Getting WKB from GEOMETRY. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] + Example 3: Getting WKB (little-endian) from GEOGRAPHY. >>> from pyspark.sql import functions as sf - >>> spark.range(1).select( - ... sf.try_reflect(sf.lit("scala.Predef"), sf.lit("require"), sf.lit(False)) - ... ).show(truncate=False) - +-----------------------------------------+ - |try_reflect(scala.Predef, require, false)| - +-----------------------------------------+ - |NULL | - +-----------------------------------------+ + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb'), 'NDR'))).collect() + [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] + + Example 4: Getting WKB (big-endian) from GEOMETRY. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb'), 'XDR'))).collect() + [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), XDR))='00000000013FF00000000000004000000000000000')] """ - return _invoke_function_over_seq_of_columns("try_reflect", cols) + if endianness is None: + return _invoke_function_over_columns("st_asbinary", geo) + else: + _endianness = lit(endianness) if isinstance(endianness, str) else endianness + return _invoke_function_over_columns("st_asbinary", geo, _endianness) @_try_remote_functions -def version() -> Column: - """ - Returns the Spark version. The string contains 2 fields, the first being a release version - and the second being a git revision. +def st_geogfromwkb(wkb: "ColumnOrName") -> Column: + """Parses the input WKB description and returns the corresponding GEOGRAPHY value. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 + + Parameters + ---------- + wkb : :class:`~pyspark.sql.Column` or str + A BINARY value in WKB format, representing a GEOGRAPHY value. + A column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.version()).show(truncate=False) # doctest: +SKIP - +----------------------------------------------+ - |version() | - +----------------------------------------------+ - |4.0.0 4f8d1f575e99aeef8990c63a9614af0fc5479330| - +----------------------------------------------+ + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] """ - return _invoke_function_over_columns("version") + return _invoke_function_over_columns("st_geogfromwkb", wkb) @_try_remote_functions -def typeof(col: "ColumnOrName") -> Column: - """ - Return DDL-formatted type string for the data type of the input. +def st_geomfromwkb( + wkb: "ColumnOrName", srid: Optional[Union["ColumnOrName", int]] = None +) -> Column: + """Parses the input WKB description and returns the corresponding GEOMETRY value. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + wkb : :class:`~pyspark.sql.Column` or str + A BINARY value in WKB format, representing a GEOMETRY value. + A column that evaluates to a binary. + srid : :class:`~pyspark.sql.Column` or int, optional + The optional SRID value of the geometry. Default is 0. + A column that evaluates to an integer. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(True, 1, 1.0, 'xyz',)], ['a', 'b', 'c', 'd']) - >>> df.select(sf.typeof(df.a), sf.typeof(df.b), sf.typeof('c'), sf.typeof('d')).show() - +---------+---------+---------+---------+ - |typeof(a)|typeof(b)|typeof(c)|typeof(d)| - +---------+---------+---------+---------+ - | boolean| bigint| double| string| - +---------+---------+---------+---------+ + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] """ - return _invoke_function_over_columns("typeof", col) + if srid is None: + return _invoke_function_over_columns("st_geomfromwkb", wkb) + else: + srid = _enum_to_value(srid) + srid = lit(srid) if isinstance(srid, int) else srid + return _invoke_function_over_columns("st_geomfromwkb", wkb, srid) @_try_remote_functions -def stack(*cols: "ColumnOrName") -> Column: - """ - Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default - unless specified otherwise. +def st_setsrid(geo: "ColumnOrName", srid: Union["ColumnOrName", int]) -> Column: + """Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a literal int for the number of rows to be separated, - and the remaining are input elements to be separated. + geo : :class:`~pyspark.sql.Column` or str + A geospatial value, either a GEOGRAPHY or a GEOMETRY. + srid : :class:`~pyspark.sql.Column` or int + An INTEGER representing the new SRID of the geospatial value. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) - >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c')).show() - +---+---+---+----+----+ - | a| b| c|col0|col1| - +---+---+---+----+----+ - | 1| 2| 3| 1| 2| - | 1| 2| 3| 3|NULL| - +---+---+---+----+----+ - - >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c').alias('x', 'y')).show() - +---+---+---+---+----+ - | a| b| c| x| y| - +---+---+---+---+----+ - | 1| 2| 3| 1| 2| - | 1| 2| 3| 3|NULL| - +---+---+---+---+----+ - >>> df.select('*', sf.stack(sf.lit(3), df.a, df.b, 'c')).show() - +---+---+---+----+ - | a| b| c|col0| - +---+---+---+----+ - | 1| 2| 3| 1| - | 1| 2| 3| 2| - | 1| 2| 3| 3| - +---+---+---+----+ + Example 1: Setting the SRID on GEOGRAPHY with SRID from another column. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'), 4326)], ['wkb', 'srid']) # noqa + >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geogfromwkb('wkb'), 'srid'))).collect() + [Row(st_srid(st_setsrid(st_geogfromwkb(wkb), srid))=4326)] - >>> df.select('*', sf.stack(sf.lit(4), df.a, df.b, 'c')).show() - +---+---+---+----+ - | a| b| c|col0| - +---+---+---+----+ - | 1| 2| 3| 1| - | 1| 2| 3| 2| - | 1| 2| 3| 3| - | 1| 2| 3|NULL| - +---+---+---+----+ + Example 2: Setting the SRID on GEOMETRY with SRID as an integer literal. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geomfromwkb('wkb'), 4326))).collect() + [Row(st_srid(st_setsrid(st_geomfromwkb(wkb, 0), 4326))=4326)] """ - return _invoke_function_over_seq_of_columns("stack", cols) + srid = _enum_to_value(srid) + srid = lit(srid) if isinstance(srid, int) else srid + return _invoke_function_over_columns("st_setsrid", geo, srid) @_try_remote_functions -def bitmap_bit_position(col: "ColumnOrName") -> Column: - """ - Returns the bit position for the given input column. +def st_srid(geo: "ColumnOrName") -> Column: + """Returns the SRID of the input GEOGRAPHY or GEOMETRY value. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column. - A column that evaluates to a long. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` + geo : :class:`~pyspark.sql.Column` or str + A geospatial value, either a GEOGRAPHY or a GEOMETRY. Examples -------- + + Example 1: Getting the SRID of GEOGRAPHY. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(123,)], ['a']) - >>> df.select('*', sf.bitmap_bit_position('a')).show() - +---+----------------------+ - | a|bitmap_bit_position(a)| - +---+----------------------+ - |123| 122| - +---+----------------------+ + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.st_srid(sf.st_geogfromwkb('wkb'))).collect() + [Row(st_srid(st_geogfromwkb(wkb))=4326)] + + Example 2: Getting the SRID of GEOMETRY. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.st_srid(sf.st_geomfromwkb('wkb'))).collect() + [Row(st_srid(st_geomfromwkb(wkb, 0))=0)] """ - return _invoke_function_over_columns("bitmap_bit_position", col) + return _invoke_function_over_columns("st_srid", geo) + + +# ---------------------- Vector Functions ---------------------- @_try_remote_functions -def bitmap_bucket_number(col: "ColumnOrName") -> Column: - """ - Returns the bucket number for the given input column. +def vector_cosine_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Returns the cosine similarity between two float vectors. + The vectors must have the same dimension. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column. - A column that evaluates to a long. + left : :class:`~pyspark.sql.Column` or column name + first vector column. + right : :class:`~pyspark.sql.Column` or column name + second vector column. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` + Returns + ------- + :class:`~pyspark.sql.Column` + cosine similarity as a float value. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(123,)], ['a']) - >>> df.select('*', sf.bitmap_bucket_number('a')).show() - +---+-----------------------+ - | a|bitmap_bucket_number(a)| - +---+-----------------------+ - |123| 1| - +---+-----------------------+ + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) + >>> df.select(sf.vector_cosine_similarity('a', 'b')).first()[0] + 0.974631... """ - return _invoke_function_over_columns("bitmap_bucket_number", col) + return _invoke_function_over_columns("vector_cosine_similarity", left, right) @_try_remote_functions -def bitmap_construct_agg(col: "ColumnOrName") -> Column: - """ - Returns a bitmap with the positions of the bits set from all the values from the input column. - The input column will most likely be bitmap_bit_position(). +def vector_inner_product(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Returns the inner product (dot product) between two float vectors. + The vectors must have the same dimension. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column will most likely be bitmap_bit_position(). - A column that evaluates to a long. + left : :class:`~pyspark.sql.Column` or column name + first vector column. + right : :class:`~pyspark.sql.Column` or column name + second vector column. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` - :meth:`pyspark.sql.functions.bitmap_and_agg` + Returns + ------- + :class:`~pyspark.sql.Column` + inner product as a float value. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,),(2,),(3,)], ["a"]) - >>> df.select( - ... sf.bitmap_construct_agg(sf.bitmap_bit_position('a')) - ... ).show() - +--------------------------------------------+ - |bitmap_construct_agg(bitmap_bit_position(a))| - +--------------------------------------------+ - | [07 00 00 00 00 0...| - +--------------------------------------------+ + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) + >>> df.select(sf.vector_inner_product('a', 'b')).first()[0] + 32.0 """ - return _invoke_function_over_columns("bitmap_construct_agg", col) + return _invoke_function_over_columns("vector_inner_product", left, right) @_try_remote_functions -def bitmap_count(col: "ColumnOrName") -> Column: - """ - Returns the number of set bits in the input bitmap. +def vector_l2_distance(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Returns the Euclidean (L2) distance between two float vectors. + The vectors must have the same dimension. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input bitmap. + left : :class:`~pyspark.sql.Column` or column name + first vector column. + right : :class:`~pyspark.sql.Column` or column name + second vector column. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_or_agg` + Returns + ------- + :class:`~pyspark.sql.Column` + L2 distance as a float value. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("FFFF",)], ["a"]) - >>> df.select(sf.bitmap_count(sf.to_binary(df.a, sf.lit("hex")))).show() - +-------------------------------+ - |bitmap_count(to_binary(a, hex))| - +-------------------------------+ - | 16| - +-------------------------------+ + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) + >>> df.select(sf.vector_l2_distance('a', 'b')).first()[0] + 5.196152... """ - return _invoke_function_over_columns("bitmap_count", col) + return _invoke_function_over_columns("vector_l2_distance", left, right) @_try_remote_functions -def bitmap_and(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """ - Returns a bitmap that is the bitwise AND of two input bitmaps. +def vector_norm(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: + """Returns the Lp norm of a float vector using the specified degree. + Degree defaults to 2.0 (Euclidean norm) if unspecified. - .. versionadded:: 4.4.0 + .. versionadded:: 4.3.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_andnot` - :meth:`pyspark.sql.functions.bitmap_or` - :meth:`pyspark.sql.functions.bitmap_xor` + vector : :class:`~pyspark.sql.Column` or column name + input vector column. + degree : :class:`~pyspark.sql.Column` or column name, optional + norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). + Defaults to 2.0. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + the Lp norm as a float value. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) - >>> df.select(sf.bitmap_and( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[70 00 00 00 00 0...| - +--------------------+ + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) + >>> df.select(sf.vector_norm('v', sf.lit(2.0).cast('float'))).first()[0] + 5.0 """ - return _invoke_function_over_columns("bitmap_and", left, right) + if degree is None: + return _invoke_function_over_columns("vector_norm", vector) + else: + return _invoke_function_over_columns("vector_norm", vector, degree) @_try_remote_functions -def bitmap_or(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """ - Returns a bitmap that is the bitwise OR of two input bitmaps. +def vector_normalize(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: + """Normalizes a float vector to unit length using the specified norm degree. + Degree defaults to 2.0 (Euclidean norm) if unspecified. - .. versionadded:: 4.4.0 + .. versionadded:: 4.3.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_and` - :meth:`pyspark.sql.functions.bitmap_andnot` - :meth:`pyspark.sql.functions.bitmap_xor` + vector : :class:`~pyspark.sql.Column` or column name + input vector column. + degree : :class:`~pyspark.sql.Column` or column name, optional + norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). + Defaults to 2.0. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + the normalized vector as an array of floats. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("10", "20")], ["left", "right"]) - >>> df.select(sf.bitmap_or( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[30 00 00 00 00 0...| - +--------------------+ + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) + >>> df.select(sf.vector_normalize('v', sf.lit(2.0).cast('float'))).first()[0] + [0.6..., 0.8...] """ - return _invoke_function_over_columns("bitmap_or", left, right) + if degree is None: + return _invoke_function_over_columns("vector_normalize", vector) + else: + return _invoke_function_over_columns("vector_normalize", vector, degree) @_try_remote_functions -def bitmap_andnot(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """ - Returns a bitmap that is the bitwise AND NOT of two input bitmaps. +def vector_avg(col: "ColumnOrName") -> Column: + """Aggregate function: returns the element-wise mean of float vectors in a group. + All vectors must have the same dimension. - .. versionadded:: 4.4.0 + .. versionadded:: 4.3.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_and` - :meth:`pyspark.sql.functions.bitmap_or` - :meth:`pyspark.sql.functions.bitmap_xor` + col : :class:`~pyspark.sql.Column` or column name + input vector column. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + the element-wise average vector as an array of floats. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) - >>> df.select(sf.bitmap_andnot( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[80 00 00 00 00 0...| - +--------------------+ + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) + >>> df.select(sf.vector_avg('v')).first()[0] + [2.0, 3.0] """ - return _invoke_function_over_columns("bitmap_andnot", left, right) + return _invoke_function_over_columns("vector_avg", col) @_try_remote_functions -def bitmap_xor(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """ - Returns a bitmap that is the bitwise XOR of two input bitmaps. +def vector_sum(col: "ColumnOrName") -> Column: + """Aggregate function: returns the element-wise sum of float vectors in a group. + All vectors must have the same dimension. - .. versionadded:: 4.4.0 + .. versionadded:: 4.3.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_and` - :meth:`pyspark.sql.functions.bitmap_andnot` - :meth:`pyspark.sql.functions.bitmap_or` + col : :class:`~pyspark.sql.Column` or column name + input vector column. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + the element-wise sum vector as an array of floats. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) - >>> df.select(sf.bitmap_xor( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[80 00 00 00 00 0...| - +--------------------+ + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) + >>> df.select(sf.vector_sum('v')).first()[0] + [4.0, 6.0] """ - return _invoke_function_over_columns("bitmap_xor", left, right) + return _invoke_function_over_columns("vector_sum", col) + + +# ---------------------- UDF, UDTF and UDT ---------------------- @_try_remote_functions -def bitmap_or_agg(col: "ColumnOrName") -> Column: +def call_udf(udfName: str, *cols: "ColumnOrName") -> Column: """ - Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. - The input column should be bitmaps created from bitmap_construct_agg(). - - .. versionadded:: 3.5.0 + Call a user-defined function. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_and_agg` + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). + udfName : str + name of the user defined function (UDF) + cols : :class:`~pyspark.sql.Column` or str + column names or :class:`~pyspark.sql.Column`\\s to be used in the UDF + + Returns + ------- + :class:`~pyspark.sql.Column` + result of executed udf. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("10",),("20",),("40",)], ["a"]) - >>> df.select(sf.bitmap_or_agg(sf.to_binary(df.a, sf.lit("hex")))).show() - +--------------------------------+ - |bitmap_or_agg(to_binary(a, hex))| - +--------------------------------+ - | [70 00 00 00 00 0...| - +--------------------------------+ + >>> from pyspark.sql.functions import call_udf, col + >>> from pyspark.sql.types import IntegerType, StringType + >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) + >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) + >>> df.select(call_udf("intX2", "id")).show() + +---------+ + |intX2(id)| + +---------+ + | 2| + | 4| + | 6| + +---------+ + >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) + >>> df.select(call_udf("strX2", col("name"))).show() + +-----------+ + |strX2(name)| + +-----------+ + | aa| + | bb| + | cc| + +-----------+ """ - return _invoke_function_over_columns("bitmap_or_agg", col) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function("call_udf", udfName, _to_seq(sc, cols, _to_java_column)) @_try_remote_functions -def bitmap_and_agg(col: "ColumnOrName") -> Column: +def unwrap_udt(col: "ColumnOrName") -> Column: """ - Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. - The input column should be bitmaps created from bitmap_construct_agg(). - - .. versionadded:: 4.1.0 + Unwrap UDT data type column into its underlying type. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` + .. versionadded:: 3.4.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). + + Returns + ------- + :class:`~pyspark.sql.Column` + The underlying representation. + + See Also + -------- + :meth:`pyspark.sql.functions.wrap_udt` Examples -------- + Example 1: Unwrap ML-specific UDT - VectorUDT + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0",),("70",),("30",)], ["a"]) - >>> df.select(sf.bitmap_and_agg(sf.to_binary(df.a, sf.lit("hex")))).show() - +---------------------------------+ - |bitmap_and_agg(to_binary(a, hex))| - +---------------------------------+ - | [30 00 00 00 00 0...| - +---------------------------------+ + >>> from pyspark.ml.linalg import Vectors + >>> vec1 = Vectors.dense(1, 2, 3) + >>> vec2 = Vectors.sparse(4, {1: 1.0, 3: 5.5}) + >>> df = spark.createDataFrame([(vec1,), (vec2,)], ["vec"]) + >>> df.select(sf.unwrap_udt("vec")).printSchema() + root + |-- unwrap_udt(vec): struct (nullable = true) + | |-- type: byte (nullable = false) + | |-- size: integer (nullable = true) + | |-- indices: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- values: array (nullable = true) + | | |-- element: double (containsNull = false) + + Example 2: Unwrap ML-specific UDT - MatrixUDT + + >>> from pyspark.sql import functions as sf + >>> from pyspark.ml.linalg import Matrices + >>> mat1 = Matrices.dense(2, 2, range(4)) + >>> mat2 = Matrices.sparse(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4]) + >>> df = spark.createDataFrame([(mat1,), (mat2,)], ["mat"]) + >>> df.select(sf.unwrap_udt("mat")).printSchema() + root + |-- unwrap_udt(mat): struct (nullable = true) + | |-- type: byte (nullable = false) + | |-- numRows: integer (nullable = false) + | |-- numCols: integer (nullable = false) + | |-- colPtrs: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- rowIndices: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- values: array (nullable = true) + | | |-- element: double (containsNull = false) + | |-- isTransposed: boolean (nullable = false) """ - return _invoke_function_over_columns("bitmap_and_agg", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("unwrap_udt", _to_java_column(col)) @_try_remote_functions -def bitmap_xor_agg(col: "ColumnOrName") -> Column: +def wrap_udt(col: "ColumnOrName", udt: "Union[UserDefinedType, Column]") -> Column: """ - Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. - The input column should be bitmaps created from bitmap_construct_agg(). + Wrap a column as a user-defined type. .. versionadded:: 4.4.0 - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` - :meth:`pyspark.sql.functions.bitmap_and_agg` - Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). + The column to wrap. The column data type must match the UDT's underlying SQL type. + udt : :class:`~pyspark.sql.types.UserDefinedType` or :class:`~pyspark.sql.Column` + The target user-defined type, or a constant string column containing its JSON + representation. + + Returns + ------- + :class:`~pyspark.sql.Column` + A column of the target user-defined type. + + See Also + -------- + :meth:`pyspark.sql.functions.unwrap_udt` Examples -------- + Example 1: Wrapping a vector struct as VectorUDT + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("10",), ("30",), ("40",)], ["a"]) - >>> df.select(sf.bitmap_xor_agg(sf.to_binary(df.a, sf.lit("hex")))).show() - +---------------------------------+ - |bitmap_xor_agg(to_binary(a, hex))| - +---------------------------------+ - | [60 00 00 00 00 0...| - +---------------------------------+ - """ - return _invoke_function_over_columns("bitmap_xor_agg", col) + >>> from pyspark.sql import Row + >>> from pyspark.sql.types import StructField, StructType + >>> from pyspark.ml.linalg import VectorUDT + >>> vector_schema = StructType([StructField("vec", VectorUDT.sqlType(), True)]) + >>> df = spark.createDataFrame( + ... [(Row(type=1, size=None, indices=None, values=[1.0, 2.0, 3.0]),)], + ... vector_schema) + >>> df.select("*", sf.wrap_udt("vec", VectorUDT())).show() + +--------------------+...+ + | vec|wrap_udt(vec...| + +--------------------+...+ + |{1, NULL, NULL, [...|...[1.0,2.0,3.0]| + +--------------------+...+ + >>> row = df.select(sf.wrap_udt("vec", VectorUDT())).first() + >>> type(row[0]) + + Example 2: Wrapping a matrix struct as MatrixUDT + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Row + >>> from pyspark.sql.types import StructField, StructType + >>> from pyspark.mllib.linalg import MatrixUDT + >>> matrix_schema = StructType([StructField("mat", MatrixUDT.sqlType(), True)]) + >>> df = spark.createDataFrame( + ... [( + ... Row( + ... type=1, + ... numRows=2, + ... numCols=2, + ... colPtrs=None, + ... rowIndices=None, + ... values=[1.0, 2.0, 3.0, 4.0], + ... isTransposed=False), + ... )], + ... matrix_schema) + >>> df.select("*", sf.wrap_udt("mat", MatrixUDT())).printSchema() + root + |-- mat: struct (nullable = true) + | |-- type: byte (nullable = false) + | |-- numRows: integer (nullable = false) + | |-- numCols: integer (nullable = false) + | |-- colPtrs: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- rowIndices: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- values: array (nullable = true) + | | |-- element: double (containsNull = false) + | |-- isTransposed: boolean (nullable = false) + |-- wrap_udt(mat...: matrix... (nullable = true) + >>> row = df.select(sf.wrap_udt("mat", MatrixUDT())).first() + >>> type(row[0]) + + """ + from pyspark.sql.classic.column import _to_java_column -# ---------------------------- User Defined Function ---------------------------------- + if isinstance(udt, _UserDefinedType): + udt_col = lit(udt.json()) + elif isinstance(udt, Column): + udt_col = udt + else: + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "UserDefinedType or Column", + "arg_name": "udt", + "arg_type": type(udt).__name__, + }, + ) + return _invoke_function("wrap_udt", _to_java_column(col), _to_java_column(udt_col)) def udaf(agg: "Aggregator") -> "UserDefinedFunctionLike": @@ -34041,230 +34304,6 @@ def arrow_udtf( return _create_pyarrow_udtf(cls=cls, returnType=returnType) -# ---------------------- Vector Functions ---------------------- - - -@_try_remote_functions -def vector_cosine_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Returns the cosine similarity between two float vectors. - The vectors must have the same dimension. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - left : :class:`~pyspark.sql.Column` or column name - first vector column. - right : :class:`~pyspark.sql.Column` or column name - second vector column. - - Returns - ------- - :class:`~pyspark.sql.Column` - cosine similarity as a float value. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) - >>> df.select(sf.vector_cosine_similarity('a', 'b')).first()[0] - 0.974631... - """ - return _invoke_function_over_columns("vector_cosine_similarity", left, right) - - -@_try_remote_functions -def vector_inner_product(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Returns the inner product (dot product) between two float vectors. - The vectors must have the same dimension. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - left : :class:`~pyspark.sql.Column` or column name - first vector column. - right : :class:`~pyspark.sql.Column` or column name - second vector column. - - Returns - ------- - :class:`~pyspark.sql.Column` - inner product as a float value. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) - >>> df.select(sf.vector_inner_product('a', 'b')).first()[0] - 32.0 - """ - return _invoke_function_over_columns("vector_inner_product", left, right) - - -@_try_remote_functions -def vector_l2_distance(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Returns the Euclidean (L2) distance between two float vectors. - The vectors must have the same dimension. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - left : :class:`~pyspark.sql.Column` or column name - first vector column. - right : :class:`~pyspark.sql.Column` or column name - second vector column. - - Returns - ------- - :class:`~pyspark.sql.Column` - L2 distance as a float value. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) - >>> df.select(sf.vector_l2_distance('a', 'b')).first()[0] - 5.196152... - """ - return _invoke_function_over_columns("vector_l2_distance", left, right) - - -@_try_remote_functions -def vector_norm(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: - """Returns the Lp norm of a float vector using the specified degree. - Degree defaults to 2.0 (Euclidean norm) if unspecified. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - vector : :class:`~pyspark.sql.Column` or column name - input vector column. - degree : :class:`~pyspark.sql.Column` or column name, optional - norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). - Defaults to 2.0. - - Returns - ------- - :class:`~pyspark.sql.Column` - the Lp norm as a float value. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) - >>> df.select(sf.vector_norm('v', sf.lit(2.0).cast('float'))).first()[0] - 5.0 - """ - if degree is None: - return _invoke_function_over_columns("vector_norm", vector) - else: - return _invoke_function_over_columns("vector_norm", vector, degree) - - -@_try_remote_functions -def vector_normalize(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: - """Normalizes a float vector to unit length using the specified norm degree. - Degree defaults to 2.0 (Euclidean norm) if unspecified. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - vector : :class:`~pyspark.sql.Column` or column name - input vector column. - degree : :class:`~pyspark.sql.Column` or column name, optional - norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). - Defaults to 2.0. - - Returns - ------- - :class:`~pyspark.sql.Column` - the normalized vector as an array of floats. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) - >>> df.select(sf.vector_normalize('v', sf.lit(2.0).cast('float'))).first()[0] - [0.6..., 0.8...] - """ - if degree is None: - return _invoke_function_over_columns("vector_normalize", vector) - else: - return _invoke_function_over_columns("vector_normalize", vector, degree) - - -@_try_remote_functions -def vector_avg(col: "ColumnOrName") -> Column: - """Aggregate function: returns the element-wise mean of float vectors in a group. - All vectors must have the same dimension. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - input vector column. - - Returns - ------- - :class:`~pyspark.sql.Column` - the element-wise average vector as an array of floats. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) - >>> df.select(sf.vector_avg('v')).first()[0] - [2.0, 3.0] - """ - return _invoke_function_over_columns("vector_avg", col) - - -@_try_remote_functions -def vector_sum(col: "ColumnOrName") -> Column: - """Aggregate function: returns the element-wise sum of float vectors in a group. - All vectors must have the same dimension. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - input vector column. - - Returns - ------- - :class:`~pyspark.sql.Column` - the element-wise sum vector as an array of floats. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) - >>> df.select(sf.vector_sum('v')).first()[0] - [4.0, 6.0] - """ - return _invoke_function_over_columns("vector_sum", col) - - def _test() -> None: import doctest diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index efa69f4154ce6..4ea5b3ef0be0c 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -53,34 +53,34 @@ import org.apache.spark.util.SparkClassUtils * only `Column` but also other types such as a native string. The other variants currently exist * for historical reasons. * - * @groupname normal_funcs Normal functions - * @groupname conditional_funcs Conditional functions - * @groupname predicate_funcs Predicate functions - * @groupname sort_funcs Sort functions - * @groupname math_funcs Mathematical functions - * @groupname string_funcs String functions - * @groupname bitwise_funcs Bitwise functions - * @groupname datetime_funcs Date and Timestamp functions - * @groupname hash_funcs Hash functions - * @groupname collection_funcs Collection functions - * @groupname array_funcs Array functions - * @groupname struct_funcs Struct functions - * @groupname map_funcs Map functions - * @groupname agg_funcs Aggregate functions - * @groupname window_funcs Window functions - * @groupname generator_funcs Generator functions - * @groupname partition_transforms Partition transform functions - * @groupname csv_funcs CSV functions - * @groupname json_funcs JSON functions - * @groupname variant_funcs VARIANT functions - * @groupname xml_funcs XML functions - * @groupname url_funcs URL functions - * @groupname misc_funcs Misc functions - * @groupname sketch_funcs Datasketch functions - * @groupname st_funcs ST geospatial functions - * @groupname vector_funcs Vector functions + * @groupname normal_funcs Normal Functions + * @groupname conditional_funcs Conditional Functions + * @groupname predicate_funcs Predicate Functions + * @groupname sort_funcs Sort Functions + * @groupname math_funcs Mathematical Functions + * @groupname string_funcs String Functions + * @groupname bitwise_funcs Bitwise Functions + * @groupname datetime_funcs Date and Timestamp Functions + * @groupname hash_funcs Hash Functions + * @groupname collection_funcs Collection Functions + * @groupname array_funcs Array Functions + * @groupname struct_funcs Struct Functions + * @groupname map_funcs Map Functions + * @groupname agg_funcs Aggregate Functions + * @groupname window_funcs Window Functions + * @groupname generator_funcs Generator Functions + * @groupname partition_transforms Partition Transformation Functions + * @groupname csv_funcs CSV Functions + * @groupname json_funcs JSON Functions + * @groupname variant_funcs VARIANT Functions + * @groupname xml_funcs XML Functions + * @groupname url_funcs URL Functions + * @groupname misc_funcs Misc Functions + * @groupname sketch_funcs Datasketch Functions + * @groupname st_funcs Geospatial ST Functions + * @groupname vector_funcs Vector Functions * @groupname udf_funcs UDF, UDAF and UDT - * @groupname Ungrouped Support functions for DataFrames + * @groupname Ungrouped Support Functions for DataFrames * @since 1.3.0 */ @Stable @@ -88,8 +88,9 @@ import org.apache.spark.util.SparkClassUtils object functions { // scalastyle:on - // Function groups are defined by the @group tags above each function and the corresponding - // @groupname declarations. Section headings in this implementation file are navigation aids. + ////////////////////////////////////////////////////////////////////////////////////////////// + // Normal Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** * Returns a [[Column]] based on the given column name. @@ -174,4675 +175,4206 @@ object functions { } } - ////////////////////////////////////////////////////////////////////////////////////////////// - // Sort functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** - * Returns a sort expression based on ascending order of the column. + * Marks a DataFrame as small enough for use in broadcast joins. + * + * The following example marks the right DataFrame for broadcast hash join using `joinKey`. * {{{ - * df.sort(asc("dept"), desc("age")) + * // left and right are DataFrames + * left.join(broadcast(right), "joinKey") * }}} * - * @group sort_funcs - * @since 1.3.0 + * @group normal_funcs + * @since 1.5.0 */ - def asc(columnName: String): Column = Column(columnName).asc + def broadcast[U](df: Dataset[U]): df.type = { + df.hint("broadcast").asInstanceOf[df.type] + } /** - * Returns a sort expression based on ascending order of the column, and null values return - * before non-null values. + * Parses the expression string into the column that it represents, similar to + * [[Dataset#selectExpr]]. * {{{ - * df.sort(asc_nulls_first("dept"), desc("age")) + * // get the number of words of each length + * df.groupBy(expr("length(word)")).count() * }}} * - * @group sort_funcs - * @since 2.1.0 + * @group normal_funcs + * @since 1.5.0 */ - def asc_nulls_first(columnName: String): Column = Column(columnName).asc_nulls_first + def expr(expr: String): Column = Column(internal.SqlExpression(expr)) /** - * Returns a sort expression based on ascending order of the column, and null values appear - * after non-null values. - * {{{ - * df.sort(asc_nulls_last("dept"), desc("age")) - * }}} + * Call a SQL function. * - * @group sort_funcs - * @since 2.1.0 + * @param funcName + * function name that follows the SQL identifier syntax (can be quoted, can be qualified) + * @param cols + * the expression parameters of function + * @group normal_funcs + * @since 3.5.0 */ - def asc_nulls_last(columnName: String): Column = Column(columnName).asc_nulls_last + @scala.annotation.varargs + def call_function(funcName: String, cols: Column*): Column = { + Column(internal.UnresolvedFunction(funcName, cols.map(_.node), isUserDefinedFunction = true)) + } + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Conditional Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Returns a sort expression based on the descending order of the column. - * {{{ - * df.sort(asc("dept"), desc("age")) - * }}} + * Returns the first column that is not null, or null if all inputs are null. * - * @group sort_funcs + * For example, `coalesce(a, b, c)` will return a if a is not null, or b if a is null and b is + * not null, or c if both a and b are null but c is not null. + * + * @param e + * the columns to work on. A column that evaluates to any type. + * @group conditional_funcs * @since 1.3.0 + * @return + * Returns a column of the same type as the input. */ - def desc(columnName: String): Column = Column(columnName).desc + @scala.annotation.varargs + def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) /** - * Returns a sort expression based on the descending order of the column, and null values appear - * before non-null values. - * {{{ - * df.sort(asc("dept"), desc_nulls_first("age")) - * }}} + * Returns col1 if it is not NaN, or col2 if col1 is NaN. * - * @group sort_funcs - * @since 2.1.0 + * Both inputs should be floating point columns (DoubleType or FloatType). + * + * @param col1 + * the first column to check. A column that evaluates to a numeric. + * @param col2 + * the column to return if the first is NaN. A column that evaluates to a numeric. + * @group conditional_funcs + * @since 1.5.0 + * @return + * Returns a column of the same type as the first input. */ - def desc_nulls_first(columnName: String): Column = Column(columnName).desc_nulls_first + def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) /** - * Returns a sort expression based on the descending order of the column, and null values appear - * after non-null values. + * Evaluates a list of conditions and returns one of multiple possible result expressions. If + * otherwise is not defined at the end, null is returned for unmatched conditions. + * * {{{ - * df.sort(asc("dept"), desc_nulls_last("age")) + * // Example: encoding gender string column into integer. + * + * // Scala: + * people.select(when(people("gender") === "male", 0) + * .when(people("gender") === "female", 1) + * .otherwise(2)) + * + * // Java: + * people.select(when(col("gender").equalTo("male"), 0) + * .when(col("gender").equalTo("female"), 1) + * .otherwise(2)) * }}} * - * @group sort_funcs - * @since 2.1.0 - */ - def desc_nulls_last(columnName: String): Column = Column(columnName).desc_nulls_last - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Aggregate functions - ////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * @group agg_funcs - * @since 1.3.0 + * @param condition + * the condition to evaluate. A column that evaluates to a boolean. + * @param value + * the value to return when the condition is true. A literal value, or a column expression. + * @group conditional_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(e: Column): Column = approx_count_distinct(e) + def when(condition: Column, value: Any): Column = + Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) /** - * @group agg_funcs - * @since 1.3.0 + * Returns `col2` if `col1` is null, or `col1` otherwise. + * + * @param col1 + * The column to test for null. A column of any type. + * @param col2 + * The column to return when col1 is null. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) + def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) /** - * @group agg_funcs - * @since 1.3.0 + * Returns null if `col1` equals to `col2`, or `col1` otherwise. + * + * @param col1 + * The value to return if it is not equal to `col2`. A column of any type. + * @param col2 + * The value compared with `col1`. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) + def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) /** - * @group agg_funcs - * @since 1.3.0 + * Returns null if `col` is equal to zero, or `col` otherwise. + * + * @param col + * The input value. A column that evaluates to a numeric. + * @group conditional_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(columnName: String, rsd: Double): Column = { - approx_count_distinct(Column(columnName), rsd) - } + def nullifzero(col: Column): Column = Column.fn("nullifzero", col) /** - * Aggregate function: returns the approximate number of distinct items in a group. + * Returns `col2` if `col1` is null, or `col1` otherwise. * - * @param e - * The column to count distinct values in. A column of any type. - * @group agg_funcs - * @since 2.1.0 + * @param col1 + * The value to return if it is not null. A column of any type. + * @param col2 + * The value to return if `col1` is null. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) + def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) /** - * Aggregate function: returns the approximate number of distinct items in a group. + * Returns `col2` if `col1` is not null, or `col3` otherwise. * - * @param columnName - * The name of the column to count distinct values in. A column of any type. - * @group agg_funcs - * @since 2.1.0 + * @param col1 + * The value that determines which branch to return. A column of any type. + * @param col2 + * The value to return if `col1` is not null. A column of any type. + * @param col3 + * The value to return if `col1` is null. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def approx_count_distinct(columnName: String): Column = approx_count_distinct( - column(columnName)) + def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) /** - * Aggregate function: returns the approximate number of distinct items in a group. - * - * @param rsd - * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a - * double. Must be a constant. + * Returns zero if `col` is null, or `col` otherwise. * - * @group agg_funcs - * @since 2.1.0 + * @param col + * The input value. A column that evaluates to a numeric. + * @group conditional_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def approx_count_distinct(e: Column, rsd: Double): Column = { - Column.fn("approx_count_distinct", e, lit(rsd)) - } + def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Predicate Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Aggregate function: returns the approximate number of distinct items in a group. - * - * @param rsd - * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a - * double. Must be a constant. + * Return true iff the column is NaN. * - * @group agg_funcs - * @since 2.1.0 + * @param e + * the column to check. A column that evaluates to a numeric. + * @group predicate_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a boolean. */ - def approx_count_distinct(columnName: String, rsd: Double): Column = { - approx_count_distinct(Column(columnName), rsd) - } + def isnan(e: Column): Column = e.isNaN /** - * Aggregate function: returns the average of the values in a group. + * Return true iff the column is null. * * @param e - * The column to average. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 1.3.0 + * the column to check. A column that evaluates to any type. + * @group predicate_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a numeric. + * Returns a column that evaluates to a boolean. */ - def avg(e: Column): Column = Column.fn("avg", e) + def isnull(e: Column): Column = e.isNull /** - * Aggregate function: returns the average of the values in a group. + * Inversion of boolean expression, i.e. NOT. + * {{{ + * // Scala: select rows that are not active (isActive === false) + * df.filter( !df("isActive") ) * - * @param columnName - * The name of the column to average. A column that evaluates to a numeric or interval. - * @group agg_funcs + * // Java: + * df.filter( not(df.col("isActive")) ); + * }}} + * + * @param e + * the column to invert. A column that evaluates to a boolean. + * @group predicate_funcs * @since 1.3.0 * @return - * Returns a column that evaluates to a numeric. + * Returns a column that evaluates to a boolean. */ - def avg(columnName: String): Column = avg(Column(columnName)) + def not(e: Column): Column = !e /** - * Aggregate function: returns a list of objects with duplicates. - * - * @param e - * The column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Returns true if `str` matches `regexp`, or false otherwise. * - * @group agg_funcs - * @since 1.6.0 + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def collect_list(e: Column): Column = Column.fn("collect_list", e) + def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) /** - * Aggregate function: returns a list of objects with duplicates. - * - * @param columnName - * The name of the column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Returns true if `str` matches `regexp`, or false otherwise. * - * @group agg_funcs - * @since 1.6.0 + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def collect_list(columnName: String): Column = collect_list(Column(columnName)) + def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) /** - * Aggregate function: returns a set of objects with duplicate elements eliminated. - * - * @param e - * The column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Returns true if `str` matches `regexp`, or false otherwise. * - * @group agg_funcs - * @since 1.6.0 + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def collect_set(e: Column): Column = Column.fn("collect_set", e) + def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) /** - * Aggregate function: returns a set of objects with duplicate elements eliminated. - * - * @param columnName - * The name of the column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Returns true if str matches `pattern` with `escapeChar`, null if any arguments are null, + * false otherwise. * - * @group agg_funcs - * @since 1.6.0 + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def collect_set(columnName: String): Column = collect_set(Column(columnName)) + def like(str: Column, pattern: Column, escapeChar: Column): Column = + Column.fn("like", str, pattern, escapeChar) /** - * Aggregate function: returns the distinct union of the elements of an array-typed column - * across rows. - * - * The aggregation buffer holds only the distinct elements, so its size is bounded by the - * element universe rather than by the number of input rows. Null elements are dropped by - * default (IGNORE NULLS), matching `collect_set`. With `RESPECT NULLS`, a single null element - * is kept, in which case this is equivalent to `array_distinct(flatten(collect_list(e)))`. The - * `RESPECT NULLS` clause is only available through SQL (e.g. - * `expr("collect_union(col) RESPECT NULLS")`). - * - * @param e - * The array column to collect the union of. A column of type array. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Returns true if str matches `pattern` with `escapeChar`('\'), null if any arguments are null, + * false otherwise. * - * @group agg_funcs - * @since 4.3.0 + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def collect_union(e: Column): Column = Column.fn("collect_union", e) + def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) /** - * Aggregate function: returns the distinct union of the elements of an array-typed column - * across rows. - * - * @param columnName - * The name of the array column to collect the union of. A column of type array. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Returns true if str matches `pattern` with `escapeChar` case-insensitively, null if any + * arguments are null, false otherwise. * - * @group agg_funcs - * @since 4.3.0 + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def collect_union(columnName: String): Column = collect_union(Column(columnName)) + def ilike(str: Column, pattern: Column, escapeChar: Column): Column = + Column.fn("ilike", str, pattern, escapeChar) /** - * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is - * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min - * sketch is a probabilistic data structure used for cardinality estimation using sub-linear - * space. + * Returns true if str matches `pattern` with `escapeChar`('\') case-insensitively, null if any + * arguments are null, false otherwise. * - * @param e - * The column to compute the sketch on. A column that evaluates to an integral, string or - * binary. - * @param eps - * The relative error, must be positive. A column that evaluates to a numeric. Must be a - * constant. - * @param confidence - * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. - * Must be a constant. - * @param seed - * The random seed. A column that evaluates to an integral. Must be a constant. - * @group agg_funcs + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @group predicate_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = - Column.fn("count_min_sketch", e, eps, confidence, seed) + def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) /** - * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is - * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min - * sketch is a probabilistic data structure used for cardinality estimation using sub-linear - * space. + * Returns true if `col` is not null, or false otherwise. * - * @param e - * The column to compute the sketch on. A column that evaluates to an integral, string or - * binary. - * @param eps - * The relative error, must be positive. A column that evaluates to a numeric. Must be a - * constant. - * @param confidence - * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. - * Must be a constant. - * @group agg_funcs - * @since 4.0.0 + * @param col + * The column to check. A column of any type. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = - count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) + def isnotnull(col: Column): Column = Column.fn("isnotnull", col) /** - * Aggregate function: returns the Pearson Correlation Coefficient for two columns. + * Returns same result as the EQUAL(=) operator for non-null operands, but returns true if both + * are null, false if one of the them is null. * - * @param column1 - * The first column. A column that evaluates to a numeric. - * @param column2 - * The second column. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param col1 + * The first column to compare. A column of any type. + * @param col2 + * The second column to compare. A column of any type. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a boolean. */ - def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) + def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Sort Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Aggregate function: returns the Pearson Correlation Coefficient for two columns. + * Returns a sort expression based on ascending order of the column. + * {{{ + * df.sort(asc("dept"), desc("age")) + * }}} * - * @param columnName1 - * The name of the first column. A column that evaluates to a numeric. - * @param columnName2 - * The name of the second column. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 - * @return - * Returns a column that evaluates to a double. + * @group sort_funcs + * @since 1.3.0 */ - def corr(columnName1: String, columnName2: String): Column = { - corr(Column(columnName1), Column(columnName2)) - } + def asc(columnName: String): Column = Column(columnName).asc /** - * Aggregate function: returns the number of items in a group. + * Returns a sort expression based on ascending order of the column, and null values return + * before non-null values. + * {{{ + * df.sort(asc_nulls_first("dept"), desc("age")) + * }}} * - * @param e - * The column to count. A column of any type. - * @group agg_funcs - * @since 1.3.0 - * @return - * Returns a column that evaluates to a long. + * @group sort_funcs + * @since 2.1.0 */ - def count(e: Column): Column = - Column.fn("count", e) + def asc_nulls_first(columnName: String): Column = Column(columnName).asc_nulls_first /** - * Aggregate function: returns the number of items in a group. + * Returns a sort expression based on ascending order of the column, and null values appear + * after non-null values. + * {{{ + * df.sort(asc_nulls_last("dept"), desc("age")) + * }}} * - * @param columnName - * The name of the column to count. A column of any type. - * @group agg_funcs - * @since 1.3.0 - * @return - * Returns a column that evaluates to a long. + * @group sort_funcs + * @since 2.1.0 */ - def count(columnName: String): TypedColumn[Any, Long] = - count(Column(columnName)).as(PrimitiveLongEncoder) + def asc_nulls_last(columnName: String): Column = Column(columnName).asc_nulls_last /** - * Aggregate function: returns the number of distinct items in a group. - * - * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. + * Returns a sort expression based on the descending order of the column. + * {{{ + * df.sort(asc("dept"), desc("age")) + * }}} * - * @param expr - * The first column. A column of any type. - * @param exprs - * Additional columns. A column of any type. - * @group agg_funcs + * @group sort_funcs * @since 1.3.0 - * @return - * Returns a column that evaluates to a long. */ - @scala.annotation.varargs - def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) + def desc(columnName: String): Column = Column(columnName).desc /** - * Aggregate function: returns the number of distinct items in a group. - * - * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. + * Returns a sort expression based on the descending order of the column, and null values appear + * before non-null values. + * {{{ + * df.sort(asc("dept"), desc_nulls_first("age")) + * }}} * - * @param columnName - * first column to compute on. A column of any type. - * @param columnNames - * additional columns to compute on. Columns of any type. - * @group agg_funcs - * @since 1.3.0 - * @return - * Returns a column that evaluates to a long. + * @group sort_funcs + * @since 2.1.0 */ - @scala.annotation.varargs - def countDistinct(columnName: String, columnNames: String*): Column = - count_distinct(Column(columnName), columnNames.map(Column.apply): _*) + def desc_nulls_first(columnName: String): Column = Column(columnName).desc_nulls_first /** - * Aggregate function: returns the number of distinct items in a group. + * Returns a sort expression based on the descending order of the column, and null values appear + * after non-null values. + * {{{ + * df.sort(asc("dept"), desc_nulls_last("age")) + * }}} * - * @param expr - * first column to compute on. A column of any type. - * @param exprs - * additional columns to compute on. Columns of any type. - * @group agg_funcs - * @since 3.2.0 - * @return - * Returns a column that evaluates to a long. + * @group sort_funcs + * @since 2.1.0 */ - @scala.annotation.varargs - def count_distinct(expr: Column, exprs: Column*): Column = - Column.fn("count", isDistinct = true, expr +: exprs: _*) + def desc_nulls_last(columnName: String): Column = Column(columnName).desc_nulls_last + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Mathematical Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Aggregate function: returns the population covariance for two columns. + * Unary minus, i.e. negate the expression. + * {{{ + * // Select the amount column and negates all values. + * // Scala: + * df.select( -df("amount") ) * - * @param column1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param column2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * // Java: + * df.select( negate(df.col("amount")) ); + * }}} + * + * @param e + * the column to negate. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def covar_pop(column1: Column, column2: Column): Column = - Column.fn("covar_pop", column1, column2) + def negate(e: Column): Column = -e /** - * Aggregate function: returns the population covariance for two columns. + * Generate a random column with independent and identically distributed (i.i.d.) samples + * uniformly distributed in [0.0, 1.0). * - * @param columnName1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param columnName2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * @param seed + * the seed for the random generator. + * @note + * The function is non-deterministic in general case. + * + * @group math_funcs + * @since 1.4.0 * @return * Returns a column that evaluates to a double. */ - def covar_pop(columnName1: String, columnName2: String): Column = { - covar_pop(Column(columnName1), Column(columnName2)) - } + def rand(seed: Long): Column = Column.fn("rand", lit(seed)) /** - * Aggregate function: returns the sample covariance for two columns. + * Generate a random column with independent and identically distributed (i.i.d.) samples + * uniformly distributed in [0.0, 1.0). * - * @param column1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param column2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * @note + * The function is non-deterministic in general case. + * + * @group math_funcs + * @since 1.4.0 * @return * Returns a column that evaluates to a double. */ - def covar_samp(column1: Column, column2: Column): Column = - Column.fn("covar_samp", column1, column2) + def rand(): Column = rand(SparkClassUtils.random.nextLong) /** - * Aggregate function: returns the sample covariance for two columns. + * Generate a column with independent and identically distributed (i.i.d.) samples from the + * standard normal distribution. * - * @param columnName1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param columnName2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * @param seed + * the seed for the random generator. + * @note + * The function is non-deterministic in general case. + * + * @group math_funcs + * @since 1.4.0 * @return * Returns a column that evaluates to a double. */ - def covar_samp(columnName1: String, columnName2: String): Column = { - covar_samp(Column(columnName1), Column(columnName2)) - } + def randn(seed: Long): Column = Column.fn("randn", lit(seed)) /** - * Aggregate function: returns the first value in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * Generate a column with independent and identically distributed (i.i.d.) samples from the + * standard normal distribution. * - * @param e - * column to fetch the first value for. A column of any type. - * @param ignoreNulls - * if first value is null then look for first non-null value. A column that evaluates to a - * boolean. Must be a constant. * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * The function is non-deterministic in general case. * - * @group agg_funcs - * @since 2.0.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def first(e: Column, ignoreNulls: Boolean): Column = - Column.fn("first", false, e, lit(ignoreNulls)) + def randn(): Column = randn(SparkClassUtils.random.nextLong) /** - * Aggregate function: returns the first value of a column in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param columnName - * column to fetch the first value for. A column of any type. - * @param ignoreNulls - * if first value is null then look for first non-null value. A column that evaluates to a - * boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Computes the square root of the specified float value. * - * @group agg_funcs - * @since 2.0.0 + * @param e + * the value to compute the square root of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def first(columnName: String, ignoreNulls: Boolean): Column = { - first(Column(columnName), ignoreNulls) - } + def sqrt(e: Column): Column = Column.fn("sqrt", e) /** - * Aggregate function: returns the first value in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * column to fetch the first value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Computes the square root of the specified float value. * - * @group agg_funcs - * @since 1.3.0 + * @param colName + * the name of a numeric column to compute the square root of. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def first(e: Column): Column = first(e, ignoreNulls = false) + def sqrt(colName: String): Column = sqrt(Column(colName)) /** - * Aggregate function: returns the first value of a column in a group. + * Returns the sum of `left` and `right` and the result is null on overflow. The acceptable + * input types are the same with the `+` operator. * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param columnName - * column to fetch the first value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 1.3.0 + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - def first(columnName: String): Column = first(Column(columnName)) + def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) /** - * Aggregate function: returns the first value in a group. - * - * @param e - * column to fetch the first value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Returns `dividend``/``divisor`. It always performs floating point division. Its result is + * always null if `divisor` is 0. * - * @group agg_funcs + * @param left + * the dividend. A column that evaluates to a numeric or interval. + * @param right + * the divisor. A column that evaluates to a numeric. + * @group math_funcs * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - def first_value(e: Column): Column = Column.fn("first_value", e) + def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) /** - * Aggregate function: returns the first value in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * column to fetch the first value for. A column of any type. - * @param ignoreNulls - * if first value is null then look for first non-null value. A column that evaluates to a - * boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Returns the remainder of `dividend``/``divisor`. Its result is always null if `divisor` is 0. * - * @group agg_funcs - * @since 3.5.0 + * @param left + * the dividend. A column that evaluates to a numeric. + * @param right + * the divisor. A column that evaluates to a numeric. + * @group math_funcs + * @since 4.0.0 * @return * Returns a column of the same type as the input. */ - def first_value(e: Column, ignoreNulls: Column): Column = - Column.fn("first_value", e, ignoreNulls) + def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) /** - * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or - * not, returns 1 for aggregated or 0 for not aggregated in the result set. + * Returns `left``*``right` and the result is null on overflow. The acceptable input types are + * the same with the `*` operator. * - * @param e - * column to check if it is aggregated. A column of any type. - * @group agg_funcs - * @since 2.0.0 + * @param left + * the multiplicand. A column that evaluates to a numeric or interval. + * @param right + * the multiplier. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a byte. + * Returns a column of the same type as the input. */ - def grouping(e: Column): Column = Column.fn("grouping", e) + def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) /** - * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or - * not, returns 1 for aggregated or 0 for not aggregated in the result set. + * Returns `left``-``right` and the result is null on overflow. The acceptable input types are + * the same with the `-` operator. * - * @param columnName - * column to check if it is aggregated. A column of any type. - * @group agg_funcs - * @since 2.0.0 + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a byte. + * Returns a column of the same type as the input. */ - def grouping(columnName: String): Column = grouping(Column(columnName)) + def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) /** - * Aggregate function: returns the level of grouping, equals to - * - * {{{ - * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) - * }}} - * - * @param cols - * columns to check for. Columns of any type. - * @note - * The list of columns should match with grouping columns exactly, or empty (means all the - * grouping columns). + * Computes the absolute value of a numeric value. * - * @group agg_funcs - * @since 2.0.0 + * @param e + * the value to compute the absolute value of. A column that evaluates to a numeric or + * interval. + * @group math_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - @scala.annotation.varargs - def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) + def abs(e: Column): Column = Column.fn("abs", e) /** - * Aggregate function: returns the level of grouping, equals to - * - * {{{ - * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) - * }}} - * - * @param colName - * the name of the first grouping column. A column of any type. - * @param colNames - * the names of the remaining grouping columns. Columns of any type. - * @note - * The list of columns should match with grouping columns exactly. - * - * @group agg_funcs - * @since 2.0.0 + * @param e + * the value to compute the inverse cosine of. A column that evaluates to a double. * @return - * Returns a column that evaluates to a long. + * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - @scala.annotation.varargs - def grouping_id(colName: String, colNames: String*): Column = { - grouping_id((Seq(colName) ++ colNames).map(n => Column(n)): _*) - } + def acos(e: Column): Column = Column.fn("acos", e) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with lgConfigK arg. - * - * @param e - * the column to compute the sketch on. A column that evaluates to an integral, a string or a - * binary. - * @param lgConfigK - * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column - * that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * @param columnName + * the value to compute the inverse cosine of. * @return - * Returns a column that evaluates to a binary. + * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def hll_sketch_agg(e: Column, lgConfigK: Column): Column = - Column.fn("hll_sketch_agg", e, lgConfigK) + def acos(columnName: String): Column = acos(Column(columnName)) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with lgConfigK arg. - * * @param e - * the column to compute the sketch on. A column that evaluates to an integral, a string or a - * binary. - * @param lgConfigK - * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column - * that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse hyperbolic cosine of. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.1.0 */ - def hll_sketch_agg(e: Column, lgConfigK: Int): Column = - Column.fn("hll_sketch_agg", e, lit(lgConfigK)) + def acosh(e: Column): Column = Column.fn("acosh", e) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with lgConfigK arg. - * * @param columnName - * the name of the column to compute the sketch on. A column that evaluates to an integral, a - * string or a binary. - * @param lgConfigK - * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column - * that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse hyperbolic cosine of. * @return - * Returns a column that evaluates to a binary. + * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 3.1.0 */ - def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { - hll_sketch_agg(Column(columnName), lgConfigK) - } + def acosh(columnName: String): Column = acosh(Column(columnName)) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with default lgConfigK value. - * * @param e - * the column to compute the sketch on. A column that evaluates to an integral, a string or a - * binary. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse sine of. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def hll_sketch_agg(e: Column): Column = - Column.fn("hll_sketch_agg", e) + def asin(e: Column): Column = Column.fn("asin", e) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with default lgConfigK value. - * * @param columnName - * the name of the column to compute the sketch on. A column that evaluates to an integral, a - * string or a binary. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse sine of. * @return - * Returns a column that evaluates to a binary. + * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def hll_sketch_agg(columnName: String): Column = { - hll_sketch_agg(Column(columnName)) - } + def asin(columnName: String): Column = asin(Column(columnName)) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values - * and allowDifferentLgConfigK is set to false. - * * @param e - * the column containing the HllSketch instances to merge. A column that evaluates to a - * binary. - * @param allowDifferentLgConfigK - * allow sketches with different lgConfigK values to be merged. A column that evaluates to a - * boolean. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse hyperbolic sine of. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. - */ - def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = - Column.fn("hll_union_agg", e, allowDifferentLgConfigK) - - /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values - * and allowDifferentLgConfigK is set to false. + * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. * - * @param e - * the column containing the HllSketch instances to merge. A column that evaluates to a - * binary. - * @param allowDifferentLgConfigK - * allow sketches with different lgConfigK values to be merged. A column that evaluates to a - * boolean. Must be a constant. - * @group agg_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a binary. + * @group math_funcs + * @since 3.1.0 */ - def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = - Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) + def asinh(e: Column): Column = Column.fn("asinh", e) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values - * and allowDifferentLgConfigK is set to false. - * * @param columnName - * the name of the column containing the HllSketch instances to merge. A column that evaluates - * to a binary. - * @param allowDifferentLgConfigK - * allow sketches with different lgConfigK values to be merged. A column that evaluates to a - * boolean. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse hyperbolic sine of. * @return - * Returns a column that evaluates to a binary. + * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 3.1.0 */ - def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { - hll_union_agg(Column(columnName), allowDifferentLgConfigK) - } + def asinh(columnName: String): Column = asinh(Column(columnName)) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. - * * @param e - * the column containing the HllSketch instances to merge. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse tangent of. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def hll_union_agg(e: Column): Column = - Column.fn("hll_union_agg", e) + def atan(e: Column): Column = Column.fn("atan", e) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. - * * @param columnName - * the name of the column containing the HllSketch instances to merge. A column that evaluates - * to a binary. - * @group agg_funcs - * @since 3.5.0 + * the value to compute the inverse tangent of. * @return - * Returns a column that evaluates to a binary. + * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def hll_union_agg(columnName: String): Column = { - hll_union_agg(Column(columnName)) - } + def atan(columnName: String): Column = atan(Column(columnName)) /** - * Aggregate function: returns the kurtosis of the values in a group. - * - * @param e - * the column to compute the kurtosis on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param y + * coordinate on y-axis. A column that evaluates to a double. + * @param x + * coordinate on x-axis. A column that evaluates to a double. * @return - * Returns a column that evaluates to a double. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def kurtosis(e: Column): Column = Column.fn("kurtosis", e) + def atan2(y: Column, x: Column): Column = Column.fn("atan2", y, x) /** - * Aggregate function: returns the kurtosis of the values in a group. - * - * @param columnName - * the name of the column to compute the kurtosis on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param y + * coordinate on y-axis + * @param xName + * coordinate on x-axis * @return - * Returns a column that evaluates to a double. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) + def atan2(y: Column, xName: String): Column = atan2(y, Column(xName)) /** - * Aggregate function: returns the last value in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * the column to take the last value from. A column of any type. - * @param ignoreNulls - * if true, returns the last non-null value; if all values are null, null is returned. A - * column that evaluates to a boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 2.0.0 + * @param yName + * coordinate on y-axis + * @param x + * coordinate on x-axis * @return - * Returns a column of the same type as the input. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def last(e: Column, ignoreNulls: Boolean): Column = - Column.fn("last", false, e, lit(ignoreNulls)) + def atan2(yName: String, x: Column): Column = atan2(Column(yName), x) /** - * Aggregate function: returns the last value of the column in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param columnName - * the name of the column to take the last value from. A column of any type. - * @param ignoreNulls - * if true, returns the last non-null value; if all values are null, null is returned. A - * column that evaluates to a boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 2.0.0 + * @param yName + * coordinate on y-axis + * @param xName + * coordinate on x-axis * @return - * Returns a column of the same type as the input. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def last(columnName: String, ignoreNulls: Boolean): Column = { - last(Column(columnName), ignoreNulls) - } + def atan2(yName: String, xName: String): Column = + atan2(Column(yName), Column(xName)) /** - * Aggregate function: returns the last value in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * column to fetch the last value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 1.3.0 + * @param y + * coordinate on y-axis + * @param xValue + * coordinate on x-axis * @return - * Returns a column of the same type as the input. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def last(e: Column): Column = last(e, ignoreNulls = false) + def atan2(y: Column, xValue: Double): Column = atan2(y, lit(xValue)) /** - * Aggregate function: returns the last value of the column in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 1.3.0 + * @param yName + * coordinate on y-axis + * @param xValue + * coordinate on x-axis * @return - * Returns a column of the same type as the input. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) + def atan2(yName: String, xValue: Double): Column = atan2(Column(yName), xValue) /** - * Aggregate function: returns the last value in a group. - * - * @param e - * column to fetch the last value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 3.5.0 + * @param yValue + * coordinate on y-axis + * @param x + * coordinate on x-axis * @return - * Returns a column of the same type as the input. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def last_value(e: Column): Column = Column.fn("last_value", e) + def atan2(yValue: Double, x: Column): Column = atan2(lit(yValue), x) /** - * Aggregate function: returns the last value in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * column to fetch the last value for. A column of any type. - * @param ignoreNulls - * whether to skip null values. A column that evaluates to a boolean. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 3.5.0 + * @param yValue + * coordinate on y-axis + * @param xName + * coordinate on x-axis * @return - * Returns a column of the same type as the input. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def last_value(e: Column, ignoreNulls: Column): Column = - Column.fn("last_value", e, ignoreNulls) + def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName)) /** - * Create time from hour, minute and second fields. For invalid inputs it will throw an error. + * @param e + * target column to compute on. A column that evaluates to a numeric. + * @return + * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. * - * @param hour - * the hour to represent, from 0 to 23. A column that evaluates to an integer. - * @param minute - * the minute to represent, from 0 to 59. A column that evaluates to an integer. - * @param second - * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. - * @group datetime_funcs - * @since 4.1.0 + * @group math_funcs + * @since 3.1.0 + */ + def atanh(e: Column): Column = Column.fn("atanh", e) + + /** + * @param columnName + * target column to compute on. * @return - * Returns a column that evaluates to a time. + * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 3.1.0 */ - def make_time(hour: Column, minute: Column, second: Column): Column = { - Column.fn("make_time", hour, minute, second) - } + def atanh(columnName: String): Column = atanh(Column(columnName)) /** - * Aggregate function: returns the most frequent value in a group. + * An expression that returns the string representation of the binary value of the given long + * column. For example, bin("12") returns "1100". * * @param e - * target column to compute on. A column of any type. - * @group agg_funcs - * @since 3.4.0 + * target column to work on. A column that evaluates to an integral. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def mode(e: Column): Column = Column.fn("mode", e) + def bin(e: Column): Column = Column.fn("bin", e) /** - * Aggregate function: returns the most frequent value in a group. - * - * When multiple values have the same greatest frequency then either any of values is returned - * if deterministic is false or is not defined, or the lowest value is returned if deterministic - * is true. + * An expression that returns the string representation of the binary value of the given long + * column. For example, bin("12") returns "1100". * - * @param e - * target column to compute on. A column of any type. - * @param deterministic - * if there are multiple equally-frequent results then return the lowest. A boolean. Must be a - * constant. - * @group agg_funcs - * @since 4.0.0 + * @param columnName + * target column to work on. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) + def bin(columnName: String): Column = bin(Column(columnName)) /** - * Aggregate function: returns the maximum value of the expression in a group. + * Computes the cube-root of the given value. * * @param e - * the target column on which the maximum value is computed. A column of any type. - * @group agg_funcs - * @since 1.3.0 + * target column to compute on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def max(e: Column): Column = Column.fn("max", e) + def cbrt(e: Column): Column = Column.fn("cbrt", e) /** - * Aggregate function: returns the maximum value of the column in a group. + * Computes the cube-root of the given column. * - * @group agg_funcs - * @since 1.3.0 + * @param columnName + * target column to compute on. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def max(columnName: String): Column = max(Column(columnName)) + def cbrt(columnName: String): Column = cbrt(Column(columnName)) /** - * Aggregate function: returns the value associated with the maximum value of ord. + * Computes the ceiling of the given value of `e` to `scale` decimal places. * * @param e - * the column representing the values to be returned. A column of any type. - * @param ord - * the column that needs to be maximized. A column of any orderable type. - * @note - * The function is non-deterministic so the output order can be different for those associated - * the same values of `e`. - * - * @group agg_funcs + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integral. Must be + * a constant. + * @group math_funcs * @since 3.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long or decimal. */ - def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) + def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) /** - * Aggregate function: returns an array of values associated with the top `k` values of `ord`. - * - * The result array contains values in descending order by their associated ordering values. - * Returns null if there are no non-null ordering values. + * Computes the ceiling of the given value of `e` to 0 decimal places. * * @param e - * the column representing the values to be returned. A column of any type. - * @param ord - * the column that needs to be maximized. A column of any orderable type. - * @param k - * the number of top values to return. An integer. Must be a constant. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. - * - * @group agg_funcs - * @since 4.2.0 + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a long or decimal. */ - def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) + def ceil(e: Column): Column = Column.fn("ceil", e) /** - * Aggregate function: returns an array of values associated with the top `k` values of `ord`. - * - * The result array contains values in descending order by their associated ordering values. - * Returns null if there are no non-null ordering values. - * - * @param e - * the column representing the values to be returned. A column of any type. - * @param ord - * the column that needs to be maximized. A column of any orderable type. - * @param k - * the number of top values to return. A column that evaluates to an integer. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. + * Computes the ceiling of the given value of `columnName` to 0 decimal places. * - * @group agg_funcs - * @since 4.2.0 + * @param columnName + * the value to compute the ceiling on. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a long or decimal. */ - def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) + def ceil(columnName: String): Column = ceil(Column(columnName)) /** - * Aggregate function: returns the average of the values in a group. Alias for avg. + * Computes the ceiling of the given value of `e` to `scale` decimal places. * * @param e - * target column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.4.0 + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integer. Must be + * a constant. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long or decimal. */ - def mean(e: Column): Column = avg(e) + def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) /** - * Aggregate function: returns the average of the values in a group. Alias for avg. + * Computes the ceiling of the given value of `e` to 0 decimal places. * - * @group agg_funcs - * @since 1.4.0 + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long or decimal. */ - def mean(columnName: String): Column = avg(columnName) + def ceiling(e: Column): Column = Column.fn("ceiling", e) /** - * Aggregate function: returns the median of the values in a group. + * Convert a number in a string column from one base to another. * - * @param e - * target column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.4.0 + * @param num + * a column to convert base for. A column that evaluates to a string. + * @param fromBase + * from base number. A column that evaluates to an integer. + * @param toBase + * to base number. A column that evaluates to an integer. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def median(e: Column): Column = Column.fn("median", e) + def conv(num: Column, fromBase: Int, toBase: Int): Column = + Column.fn("conv", num, lit(fromBase), lit(toBase)) /** - * Aggregate function: returns the minimum value of the expression in a group. - * * @param e - * the target column on which the minimum value is computed. A column of any type. - * @group agg_funcs - * @since 1.3.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column of the same type as the input. + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def min(e: Column): Column = Column.fn("min", e) + def cos(e: Column): Column = Column.fn("cos", e) /** - * Aggregate function: returns the minimum value of the column in a group. - * * @param columnName - * the name of the column on which the minimum value is computed. A column of an orderable - * type. - * @group agg_funcs - * @since 1.3.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column of the same type as the input. + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def min(columnName: String): Column = min(Column(columnName)) + def cos(columnName: String): Column = cos(Column(columnName)) /** - * Aggregate function: returns the value associated with the minimum value of ord. - * * @param e - * the column representing the values that will be returned. A column of any type. - * @param ord - * the column that needs to be minimized. A column of an orderable type. - * @note - * The function is non-deterministic so the output order can be different for those associated - * the same values of `e`. - * - * @group agg_funcs - * @since 3.3.0 + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column of the same type as the input. + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) + def cosh(e: Column): Column = Column.fn("cosh", e) /** - * Aggregate function: returns an array of values associated with the bottom `k` values of - * `ord`. - * - * The result array contains values in ascending order by their associated ordering values. - * Returns null if there are no non-null ordering values. - * - * @param e - * the column representing the values that will be returned. A column of any type. - * @param ord - * the column that needs to be minimized. A column of an orderable type. - * @param k - * the number of bottom values to return. An integer. Must be a constant. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. - * - * @group agg_funcs - * @since 4.2.0 + * @param columnName + * hyperbolic angle * @return - * Returns a column that evaluates to an array. + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) + def cosh(columnName: String): Column = cosh(Column(columnName)) /** - * Aggregate function: returns an array of values associated with the bottom `k` values of - * `ord`. - * - * The result array contains values in ascending order by their associated ordering values. - * Returns null if there are no non-null ordering values. - * * @param e - * the column representing the values that will be returned. A column of any type. - * @param ord - * the column that needs to be minimized. A column of an orderable type. - * @param k - * the number of bottom values to return. A column that evaluates to an integral. Must be a - * constant. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. - * - * @group agg_funcs - * @since 4.2.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to an array. + * cotangent of the angle. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.3.0 */ - def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) + def cot(e: Column): Column = Column.fn("cot", e) /** - * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given - * percentage(s) with value range in [0.0, 1.0]. - * * @param e - * the column to compute the percentile on. A column that evaluates to a numeric or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a double. + * cosecant of the angle. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.3.0 */ - def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) + def csc(e: Column): Column = Column.fn("csc", e) /** - * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given - * percentage(s) with value range in [0.0, 1.0]. + * Returns Euler's number. * - * @param e - * the column to compute the percentile on. A column that evaluates to a numeric or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @param frequency - * the positive frequency with which to weight each value. A column that evaluates to an - * integral. - * @group agg_funcs + * @group math_funcs * @since 3.5.0 * @return * Returns a column that evaluates to a double. */ - def percentile(e: Column, percentage: Column, frequency: Column): Column = - Column.fn("percentile", e, percentage, frequency) + def e(): Column = Column.fn("e") /** - * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is - * the smallest value in the ordered `col` values (sorted from least to greatest) such that no - * more than `percentage` of `col` values is less than the value or equal to that value. - * - * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating - * point value, it must be between 0.0 and 1.0. - * - * The accuracy parameter is a positive numeric literal which controls approximation accuracy at - * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the - * relative error of the approximation. + * Computes the exponential of the given value. * * @param e - * the column to compute the approximate percentile on. A column that evaluates to a numeric - * or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @param accuracy - * a positive numeric literal that controls approximation accuracy at the cost of memory. A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.1.0 + * target column to compute on. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = - Column.fn("percentile_approx", e, percentage, accuracy) + def exp(e: Column): Column = Column.fn("exp", e) /** - * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is - * the smallest value in the ordered `col` values (sorted from least to greatest) such that no - * more than `percentage` of `col` values is less than the value or equal to that value. - * - * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating - * point value, it must be between 0.0 and 1.0. - * - * The accuracy parameter is a positive numeric literal which controls approximation accuracy at - * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the - * relative error of the approximation. + * Computes the exponential of the given column. * - * @param e - * the column to compute the approximate percentile on. A column that evaluates to a numeric - * or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @param accuracy - * a positive numeric literal that controls approximation accuracy at the cost of memory. A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * @param columnName + * target column to compute on. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { - Column.fn("approx_percentile", e, percentage, accuracy) - } + def exp(columnName: String): Column = exp(Column(columnName)) /** - * Aggregate function: returns the product of all numerical elements in a group. + * Computes the exponential of the given value minus one. * * @param e - * the column to compute the product on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.2.0 + * column to calculate exponential for. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return * Returns a column that evaluates to a double. */ - def product(e: Column): Column = Column.internalFn("product", e) + def expm1(e: Column): Column = Column.fn("expm1", e) /** - * Aggregate function: returns the skewness of the values in a group. + * Computes the exponential of the given column minus one. * - * @param e - * the column to compute the skewness on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param columnName + * column name to calculate exponential for. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return * Returns a column that evaluates to a double. */ - def skewness(e: Column): Column = Column.fn("skewness", e) + def expm1(columnName: String): Column = expm1(Column(columnName)) /** - * Aggregate function: returns the skewness of the values in a group. + * Computes the factorial of the given value. * - * @param columnName - * the name of the column to compute the skewness on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param e + * a column to calculate factorial for. A column that evaluates to an integral. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long. */ - def skewness(columnName: String): Column = skewness(Column(columnName)) + def factorial(e: Column): Column = Column.fn("factorial", e) /** - * Aggregate function: alias for `stddev_samp`. + * Computes the floor of the given value of `e` to `scale` decimal places. * * @param e - * the column to compute the standard deviation on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * the target column to compute the floor on. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to control the rounding behavior. A column that evaluates to + * an integral. + * @group math_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long or decimal. */ - def std(e: Column): Column = Column.fn("std", e) + def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) /** - * Aggregate function: alias for `stddev_samp`. + * Computes the floor of the given value of `e` to 0 decimal places. * * @param e - * the column to compute the standard deviation on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * the target column to compute the floor on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long or decimal. */ - def stddev(e: Column): Column = Column.fn("stddev", e) + def floor(e: Column): Column = Column.fn("floor", e) /** - * Aggregate function: alias for `stddev_samp`. + * Computes the floor of the given column value to 0 decimal places. * * @param columnName - * the name of the column to compute the standard deviation on. A column that evaluates to a - * numeric. - * @group agg_funcs - * @since 1.6.0 + * the target column name to compute the floor on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long or decimal. */ - def stddev(columnName: String): Column = stddev(Column(columnName)) + def floor(columnName: String): Column = floor(Column(columnName)) /** - * Aggregate function: returns the sample standard deviation of the expression in a group. + * Returns the greatest value of the list of values, skipping null values. This function takes + * at least 2 parameters. It will return null iff all parameters are null. * - * @param e - * the column to compute the sample standard deviation on. A column that evaluates to a - * numeric. - * @group agg_funcs - * @since 1.6.0 - * @return - * Returns a column that evaluates to a double. + * @param exprs + * columns to check for greatest value. A column that evaluates to any type. + * @group math_funcs + * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ - def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) + @scala.annotation.varargs + def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) /** - * Aggregate function: returns the sample standard deviation of the expression in a group. + * Returns the greatest value of the list of column names, skipping null values. This function + * takes at least 2 parameters. It will return null iff all parameters are null. * * @param columnName - * Name of the column to compute the sample standard deviation on. A column that evaluates to - * a numeric. - * @group agg_funcs - * @since 1.6.0 + * the first column name to check for greatest value. A column of a comparable type. + * @param columnNames + * the remaining column names to check for greatest value. Columns of a comparable type. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) + @scala.annotation.varargs + def greatest(columnName: String, columnNames: String*): Column = { + greatest((columnName +: columnNames).map(Column.apply): _*) + } /** - * Aggregate function: returns the population standard deviation of the expression in a group. + * Computes hex value of the given column. * - * @param e - * The column to compute the population standard deviation on. A column that evaluates to a - * numeric. - * @group agg_funcs - * @since 1.6.0 + * @param column + * target column to work on. A column that evaluates to an integral, string or binary. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) + def hex(column: Column): Column = Column.fn("hex", column) /** - * Aggregate function: returns the population standard deviation of the expression in a group. + * Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to + * the byte representation of number. * - * @param columnName - * Name of the column to compute the population standard deviation on. A column that evaluates - * to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param column + * target column to work on. A column that evaluates to a string. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) + def unhex(column: Column): Column = Column.fn("unhex", column) /** - * Aggregate function: returns the sum of all values in the expression. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param e - * The column to sum. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 1.3.0 + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a double. */ - def sum(e: Column): Column = Column.fn("sum", e) + def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) /** - * Aggregate function: returns the sum of all values in the given column. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param columnName - * Name of the column to sum. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 1.3.0 + * @param l + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a double. */ - def sum(columnName: String): Column = sum(Column(columnName)) + def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) /** - * Aggregate function: returns the sum of distinct values in the expression. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @group agg_funcs - * @since 1.3.0 + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a double. */ - @deprecated("Use sum_distinct", "3.2.0") - def sumDistinct(e: Column): Column = sum_distinct(e) + def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) /** - * Aggregate function: returns the sum of distinct values in the expression. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @group agg_funcs - * @since 1.3.0 + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a double. */ - @deprecated("Use sum_distinct", "3.2.0") - def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) + def hypot(leftName: String, rightName: String): Column = + hypot(Column(leftName), Column(rightName)) /** - * Aggregate function: returns the sum of distinct values in the expression. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param e - * The column to sum distinct values of. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 3.2.0 + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. Must be a constant. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a double. */ - def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) + def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input - * column via a Datasketches Intersection instance. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param e - * The column of Datasketches ThetaSketch instances to intersect. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.0 + * @param leftName + * The a leg of the triangle. A column that evaluates to a numeric. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_intersection_agg(e: Column): Column = - Column.fn("theta_intersection_agg", e) + def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input - * volumn via a Datasketches Intersection instance. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param columnName - * Name of the column of Datasketches ThetaSketch instances to intersect. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.1.0 + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_intersection_agg(columnName: String): Column = - theta_intersection_agg(Column(columnName)) + def hypot(l: Double, r: Column): Column = hypot(lit(l), r) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the `lgNomEntries` nominal - * entries. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param e - * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, - * binary or array. - * @param lgNomEntries - * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and - * 26). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param rightName + * The b leg of the triangle. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = - Column.fn("theta_sketch_agg", e, lgNomEntries) + def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the `lgNomEntries` nominal - * entries. + * Returns the least value of the list of values, skipping null values. This function takes at + * least 2 parameters. It will return null iff all parameters are null. * - * @param e - * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, - * binary or array. - * @param lgNomEntries - * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and - * 26). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param exprs + * The values to be compared. Columns that evaluate to a comparable type. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = - Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) + @scala.annotation.varargs + def least(exprs: Column*): Column = Column.fn("least", exprs: _*) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the `lgNomEntries` nominal - * entries. + * Returns the least value of the list of column names, skipping null values. This function + * takes at least 2 parameters. It will return null iff all parameters are null. * * @param columnName - * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, - * string, binary or array. - * @param lgNomEntries - * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and - * 26). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * The name of the first column to be compared. A column of a comparable type. + * @param columnNames + * The names of the remaining columns to be compared. Columns of a comparable type. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = - theta_sketch_agg(Column(columnName), lgNomEntries) + @scala.annotation.varargs + def least(columnName: String, columnNames: String*): Column = { + least((columnName +: columnNames).map(Column.apply): _*) + } /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the default value of 12 for - * `lgNomEntries`. + * Computes the natural logarithm of the given value. * * @param e - * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, - * binary or array. - * @group agg_funcs - * @since 4.1.0 + * The value to compute the natural logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_sketch_agg(e: Column): Column = - Column.fn("theta_sketch_agg", e) + def ln(e: Column): Column = Column.fn("ln", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the default value of 12 for - * `lgNomEntries`. + * Computes the natural logarithm of the given value. * - * @param columnName - * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, - * string, binary or array. - * @group agg_funcs - * @since 4.1.0 + * @param e + * The value to compute the natural logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_sketch_agg(columnName: String): Column = - theta_sketch_agg(Column(columnName)) + def log(e: Column): Column = ln(e) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Computes the natural logarithm of the given column. * - * @param e - * The column containing binary ThetaSketch representations. A column that evaluates to a - * binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName + * The name of the column to compute the natural logarithm of. A column that evaluates to a + * numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union_agg(e: Column, lgNomEntries: Column): Column = - Column.fn("theta_union_agg", e, lgNomEntries) + def log(columnName: String): Column = log(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Returns the first argument-base logarithm of the second argument. * - * @param e - * The column containing binary ThetaSketch representations. A column that evaluates to a - * binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. + * @param a + * The value to compute the logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union_agg(e: Column, lgNomEntries: Int): Column = - Column.fn("theta_union_agg", e, lit(lgNomEntries)) + def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Returns the first argument-base logarithm of the second argument. * + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. * @param columnName - * The name of the column containing binary ThetaSketch representations. A column that - * evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * The name of the column to compute the logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union_agg(columnName: String, lgNomEntries: Int): Column = - theta_union_agg(Column(columnName), lgNomEntries) + def log(base: Double, columnName: String): Column = log(base, Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It is configured with the default value of 12 for - * `lgNomEntries`. + * Computes the logarithm of the given value in base 10. * * @param e - * The column containing binary ThetaSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.0 + * The value to compute the base-10 logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union_agg(e: Column): Column = - Column.fn("theta_union_agg", e) + def log10(e: Column): Column = Column.fn("log10", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It is configured with the default value of 12 for - * `lgNomEntries`. + * Computes the logarithm of the given value in base 10. * * @param columnName - * The name of the column containing binary ThetaSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.1.0 + * The name of the column to compute the base-10 logarithm of. A column that evaluates to a + * numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union_agg(columnName: String): Column = - theta_union_agg(Column(columnName)) + def log10(columnName: String): Column = log10(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. The mode parameter specifies - * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). + * Computes the natural logarithm of the given value plus one. * * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * The value to compute the natural logarithm of the value plus one. A column that evaluates + * to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_double(e: Column, mode: Column): Column = - Column.fn("tuple_intersection_agg_double", e, mode) + def log1p(e: Column): Column = Column.fn("log1p", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. The mode parameter specifies - * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). + * Computes the natural logarithm of the given column plus one. * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param columnName + * The name of the column to compute the natural logarithm of the value plus one. A column + * that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_double(e: Column, mode: String): Column = - Column.fn("tuple_intersection_agg_double", e, lit(mode)) + def log1p(columnName: String): Column = log1p(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. The mode parameter specifies - * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). + * Computes the logarithm of the given column in base 2. + * + * @param expr + * The value to compute the base-2 logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. + */ + def log2(expr: Column): Column = Column.fn("log2", expr) + + /** + * Computes the logarithm of the given value in base 2. * * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * a column to calculate logarithm for. A column that evaluates to a double. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_double(columnName: String, mode: String): Column = - tuple_intersection_agg_double(Column(columnName), mode) + def log2(columnName: String): Column = log2(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. It is configured with the - * default mode of 'sum'. + * Returns the negated value. * * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.2.0 + * column to calculate negative value for. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_intersection_agg_double(e: Column): Column = - Column.fn("tuple_intersection_agg_double", e) + def negative(e: Column): Column = Column.fn("negative", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. It is configured with the - * default mode of 'sum'. + * Returns Pi. * - * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.2.0 + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_double(columnName: String): Column = - tuple_intersection_agg_double(Column(columnName)) + def pi(): Column = Column.fn("pi") /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Returns the value. * * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a binary. + * input value column. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ - def tuple_intersection_agg_integer(e: Column, mode: Column): Column = - Column.fn("tuple_intersection_agg_integer", e, mode) + def positive(e: Column): Column = Column.fn("positive", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Returns the value of the first argument raised to the power of the second argument. * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(e: Column, mode: String): Column = - Column.fn("tuple_intersection_agg_integer", e, lit(mode)) + def pow(l: Column, r: Column): Column = Column.fn("power", l, r) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Returns the value of the first argument raised to the power of the second argument. * - * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param l + * the base number. A column that evaluates to a double. + * @param rightName + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(columnName: String, mode: String): Column = - tuple_intersection_agg_integer(Column(columnName), mode) + def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. It is configured with - * the default mode of 'sum'. + * Returns the value of the first argument raised to the power of the second argument. * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.2.0 + * @param leftName + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(e: Column): Column = - Column.fn("tuple_intersection_agg_integer", e) + def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. It is configured with - * the default mode of 'sum'. + * Returns the value of the first argument raised to the power of the second argument. * - * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.2.0 + * @param leftName + * the base number. + * @param rightName + * the exponent number. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(columnName: String): Column = - tuple_intersection_agg_integer(Column(columnName)) + def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Returns the value of the first argument raised to the power of the second argument. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - key: Column, - summary: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_sketch_agg_double", key, summary, lgNomEntries, mode) + def pow(l: Column, r: Double): Column = pow(l, lit(r)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Returns the value of the first argument raised to the power of the second argument. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param leftName + * the base number. + * @param r + * the exponent number. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - key: Column, - summary: Column, - lgNomEntries: Int, - mode: String): Column = - Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries), lit(mode)) + def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Returns the value of the first argument raised to the power of the second argument. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to a numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int, - mode: String): Column = - tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) + def pow(l: Double, r: Column): Column = pow(lit(l), r) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Returns the value of the first argument raised to the power of the second argument. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param l + * the base number. + * @param rightName + * the exponent number. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = - Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) + def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Returns the value of the first argument raised to the power of the second argument. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to a numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int): Column = - tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) + def power(l: Column, r: Column): Column = Column.fn("power", l, r) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Returns the positive value of dividend mod divisor. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @group agg_funcs - * @since 4.2.0 + * @param dividend + * the column that contains dividend, or the specified dividend value. A column that evaluates + * to a numeric. + * @param divisor + * the column that contains divisor, or the specified divisor value. A column that evaluates + * to a numeric. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_sketch_agg_double(key: Column, summary: Column): Column = - Column.fn("tuple_sketch_agg_double", key, summary) + def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Returns the double value that is closest in value to the argument and is equal to a + * mathematical integer. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.2.0 + * @param e + * target column to compute on. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = - tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) + def rint(e: Column): Column = Column.fn("rint", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Returns the double value that is closest in value to the argument and is equal to a + * mathematical integer. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param columnName + * the numeric column name to round to the closest integer. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_integer( - key: Column, - summary: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_sketch_agg_integer", key, summary, lgNomEntries, mode) + def rint(columnName: String): Column = rint(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Returns the value of the column `e` rounded to 0 decimal places with HALF_UP round mode. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to round. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_sketch_agg_integer( - key: Column, - summary: Column, - lgNomEntries: Int, - mode: String): Column = - Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries), lit(mode)) + def round(e: Column): Column = round(e, 0) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to an integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_sketch_agg_integer( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int, - mode: String): Column = - tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) + def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = - Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) + def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Truncates the value of `e` toward zero to 0 decimal places. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to an integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to truncate. A column that evaluates to a numeric. * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input, except that a decimal input may return a + * decimal of different precision and scale. + * @group math_funcs + * @since 4.4.0 */ - def tuple_sketch_agg_integer( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int): Column = - tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) + def truncate(e: Column): Column = truncate(e, 0) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than + * or equal to 0, or to the left of the decimal point when `scale` is less than 0. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to truncate. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to keep. A column that evaluates to an integral. Must be a + * constant. * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input, except that a decimal input may return a + * decimal of different precision and scale. + * @group math_funcs + * @since 4.4.0 */ - def tuple_sketch_agg_integer(key: Column, summary: Column): Column = - Column.fn("tuple_sketch_agg_integer", key, summary) + def truncate(e: Column, scale: Int): Column = Column.fn("truncate", e, lit(scale)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than + * or equal to 0, or to the left of the decimal point when `scale` is less than 0. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to an integral. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to truncate. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to keep. A column that evaluates to an integral. Must be a + * constant. * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input, except that a decimal input may return a + * decimal of different precision and scale. + * @group math_funcs + * @since 4.4.0 */ - def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = - tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) + def truncate(e: Column, scale: Column): Column = Column.fn("truncate", e, scale) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Returns the value of the column `e` rounded to 0 decimal places with HALF_EVEN round mode. * * @param e - * the column containing binary TupleSketch representations to union. A column that evaluates - * to a binary. - * @param lgNomEntries - * the log-base-2 of nominal entries for the union buffer (must be between 4 and 26). A column - * that evaluates to an integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * the value to round. A column that evaluates to a numeric. + * @group math_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) + def bround(e: Column): Column = bround(e, 0) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) + def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = - tuple_union_agg_double(Column(columnName), lgNomEntries, mode) + def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. - * * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * secant of the angle. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.3.0 */ - def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) + def sec(e: Column): Column = Column.fn("sec", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. + * Computes the signum of the given value. * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * @param e + * the value to compute the signum of. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = - tuple_union_agg_double(Column(columnName), lgNomEntries) + def sign(e: Column): Column = Column.fn("sign", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. + * Computes the signum of the given value. * * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @group agg_funcs - * @since 4.2.0 + * the value to compute the signum of. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_union_agg_double(e: Column): Column = - Column.fn("tuple_union_agg_double", e) + def signum(e: Column): Column = Column.fn("signum", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. + * Computes the signum of the given column. * * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.2.0 + * column to compute the signum on. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_union_agg_double(columnName: String): Column = - tuple_union_agg_double(Column(columnName)) + def signum(columnName: String): Column = signum(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). - * * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. - */ - def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) - - /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a binary. + * @group math_funcs + * @since 1.4.0 */ - def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) + def sin(e: Column): Column = Column.fn("sin", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). - * * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. + * @group math_funcs + * @since 1.4.0 */ - def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = - tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) + def sin(columnName: String): Column = sin(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. - * * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) + def sinh(e: Column): Column = Column.fn("sinh", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. - * * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs - * @since 4.2.0 + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = - tuple_union_agg_integer(Column(columnName), lgNomEntries) + def sinh(columnName: String): Column = sinh(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. - * * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @group agg_funcs - * @since 4.2.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def tuple_union_agg_integer(e: Column): Column = - Column.fn("tuple_union_agg_integer", e) + def tan(e: Column): Column = Column.fn("tan", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. - * * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.2.0 + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def tuple_union_agg_integer(columnName: String): Column = - tuple_union_agg_integer(Column(columnName)) + def tan(columnName: String): Column = tan(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). - * * @param e - * The input column containing the values to aggregate. A column that evaluates to an - * integral. - * @param k - * The parameter that controls the size and accuracy of the sketch. A column that evaluates to - * an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def kll_sketch_agg_bigint(e: Column, k: Column): Column = - Column.fn("kll_sketch_agg_bigint", e, k) + def tanh(e: Column): Column = Column.fn("tanh", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). - * - * @param e - * The input column containing the values to aggregate. A column that evaluates to an - * integral. - * @param k - * The parameter that controls the size and accuracy of the sketch. A column that evaluates to - * an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def kll_sketch_agg_bigint(e: Column, k: Int): Column = - Column.fn("kll_sketch_agg_bigint", e, lit(k)) + def tanh(columnName: String): Column = tanh(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). - * - * @param columnName - * The column containing bigint values to aggregate. A column that evaluates to an integral. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def kll_sketch_agg_bigint(columnName: String, k: Int): Column = - kll_sketch_agg_bigint(Column(columnName), k) + @deprecated("Use degrees", "2.1.0") + def toDegrees(e: Column): Column = degrees(e) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column with default k value of 200. - * - * @param e - * The column containing bigint values to aggregate. A column that evaluates to an integral. - * @group agg_funcs - * @since 4.1.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def kll_sketch_agg_bigint(e: Column): Column = - Column.fn("kll_sketch_agg_bigint", e) + @deprecated("Use degrees", "2.1.0") + def toDegrees(columnName: String): Column = degrees(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column with default k value of 200. + * Converts an angle measured in radians to an approximately equivalent angle measured in + * degrees. * - * @param columnName - * The column containing bigint values to aggregate. A column that evaluates to an integral. - * @group agg_funcs - * @since 4.1.0 + * @param e + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 2.1.0 */ - def kll_sketch_agg_bigint(columnName: String): Column = - kll_sketch_agg_bigint(Column(columnName)) + def degrees(e: Column): Column = Column.fn("degrees", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Converts an angle measured in radians to an approximately equivalent angle measured in + * degrees. * - * @param e - * The column containing float values to aggregate. A column that evaluates to a float. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 2.1.0 */ - def kll_sketch_agg_float(e: Column, k: Column): Column = - Column.fn("kll_sketch_agg_float", e, k) + def degrees(columnName: String): Column = degrees(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). - * - * @param e - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def kll_sketch_agg_float(e: Column, k: Int): Column = - Column.fn("kll_sketch_agg_float", e, lit(k)) + @deprecated("Use radians", "2.1.0") + def toRadians(e: Column): Column = radians(e) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). - * - * @param columnName - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def kll_sketch_agg_float(columnName: String, k: Int): Column = - kll_sketch_agg_float(Column(columnName), k) + @deprecated("Use radians", "2.1.0") + def toRadians(columnName: String): Column = radians(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column with default k value of 200. + * Converts an angle measured in degrees to an approximately equivalent angle measured in + * radians. * * @param e - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * angle in degrees. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 2.1.0 */ - def kll_sketch_agg_float(e: Column): Column = - Column.fn("kll_sketch_agg_float", e) + def radians(e: Column): Column = Column.fn("radians", e) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column with default k value of 200. + * Converts an angle measured in degrees to an approximately equivalent angle measured in + * radians. * * @param columnName - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * angle in degrees. A column that evaluates to a double. * @return - * Returns a column that evaluates to a binary. + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 2.1.0 */ - def kll_sketch_agg_float(columnName: String): Column = - kll_sketch_agg_float(Column(columnName)) + def radians(columnName: String): Column = radians(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Returns the bucket number into which the value of this expression would fall after being + * evaluated. Note that input arguments must follow conditions listed below; otherwise, the + * method will return null. * - * @param e - * The column containing double values to aggregate. A column that evaluates to a float or - * double. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param v + * value to compute a bucket number in the histogram. A column that evaluates to a double or + * interval. + * @param min + * minimum value of the histogram. A column that evaluates to a double or interval. + * @param max + * maximum value of the histogram. A column that evaluates to a double or interval. + * @param numBucket + * the number of buckets. A column that evaluates to a long. * @return - * Returns a column that evaluates to a binary. + * the bucket number into which the value would fall after being evaluated. Returns a column + * that evaluates to a long. + * @group math_funcs + * @since 3.5.0 */ - def kll_sketch_agg_double(e: Column, k: Column): Column = - Column.fn("kll_sketch_agg_double", e, k) + def width_bucket(v: Column, min: Column, max: Column, numBucket: Column): Column = + Column.fn("width_bucket", v, min, max, numBucket) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Returns a random value with independent and identically distributed (i.i.d.) values with the + * specified range of numbers. The provided numbers specifying the minimum and maximum values of + * the range must be constant. If both of these numbers are integers, then the result will also + * be an integer. Otherwise if one or both of these are floating-point numbers, then the result + * will also be a floating-point number. * - * @param e - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def kll_sketch_agg_double(e: Column, k: Int): Column = - Column.fn("kll_sketch_agg_double", e, lit(k)) + def uniform(min: Column, max: Column): Column = + uniform(min, max, lit(SparkClassUtils.random.nextLong)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Returns a random value with independent and identically distributed (i.i.d.) values with the + * specified range of numbers, with the chosen random seed. The provided numbers specifying the + * minimum and maximum values of the range must be constant. If both of these numbers are + * integers, then the result will also be an integer. Otherwise if one or both of these are + * floating-point numbers, then the result will also be a floating-point number. * - * @param columnName - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def kll_sketch_agg_double(columnName: String, k: Int): Column = - kll_sketch_agg_double(Column(columnName), k) + def uniform(min: Column, max: Column, seed: Column): Column = + Column.fn("uniform", min, max, seed) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column with default k value of 200. + * Returns a random value with independent and identically distributed (i.i.d.) uniformly + * distributed values in [0, 1). * - * @param e - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def kll_sketch_agg_double(e: Column): Column = - Column.fn("kll_sketch_agg_double", e) + def random(seed: Column): Column = Column.fn("random", seed) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column with default k value of 200. + * Returns a random value with independent and identically distributed (i.i.d.) uniformly + * distributed values in [0, 1). * - * @param columnName - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def kll_sketch_agg_double(columnName: String): Column = - kll_sketch_agg_double(Column(columnName)) + def random(): Column = random(lit(SparkClassUtils.random.nextLong)) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // String Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range - * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Returns a string of the specified length whose characters are chosen uniformly at random from + * the following pool of characters: 0-9, a-z, A-Z. The string length must be a constant + * two-byte or four-byte integer (SMALLINT or INT, respectively). * - * @param e - * The column containing binary KllLongsSketch representations to merge. A column that - * evaluates to a binary. - * @param k - * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_bigint(e: Column, k: Column): Column = - Column.fn("kll_merge_agg_bigint", e, k) + def randstr(length: Column): Column = + randstr(length, lit(SparkClassUtils.random.nextLong)) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range - * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Returns a string of the specified length whose characters are chosen uniformly at random from + * the following pool of characters: 0-9, a-z, A-Z, with the chosen random seed. The string + * length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). * - * @param e - * The column containing binary KllLongsSketch representations to merge. A column that - * evaluates to a binary. - * @param k - * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. + * @param seed + * the random seed to use. A column that evaluates to an integral. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_bigint(e: Column, k: Int): Column = - Column.fn("kll_merge_agg_bigint", e, lit(k)) + def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range - * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Computes the numeric value of the first character of the string column, and returns the + * result as an int column. * - * @param columnName - * The column containing binary KllLongsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integral. - * @group agg_funcs - * @since 4.1.2 + * @param e + * The target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an integer. */ - def kll_merge_agg_bigint(columnName: String, k: Int): Column = - kll_merge_agg_bigint(Column(columnName), k) + def ascii(e: Column): Column = Column.fn("ascii", e) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Computes the BASE64 encoding of a binary column and returns it as a string column. This is + * the reverse of unbase64. * * @param e - * The column containing binary KllLongsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * The target column to work on. A column that evaluates to a binary. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_bigint(e: Column): Column = - Column.fn("kll_merge_agg_bigint", e) + def base64(e: Column): Column = Column.fn("base64", e) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a string column. + * This is the reverse of from_base32. * - * @param columnName - * The column containing binary KllLongsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param e + * The target column to work on. A column that evaluates to a binary. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_bigint(columnName: String): Column = - kll_merge_agg_bigint(Column(columnName)) + def to_base32(e: Column): Column = Column.fn("to_base32", e) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Calculates the bit length for the specified string column. * * @param e - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integral. - * @group agg_funcs - * @since 4.1.2 + * The source column or strings. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an integer. */ - def kll_merge_agg_float(e: Column, k: Column): Column = - Column.fn("kll_merge_agg_float", e, k) + def bit_length(e: Column): Column = Column.fn("bit_length", e) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Concatenates multiple input string columns together into a single string column, using the + * given separator. * - * @param e - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param sep + * The words separator. A column that evaluates to a string. Must be a constant. + * @param exprs + * The list of columns to work on. Each a column that evaluates to a string or an array of + * strings. + * @note + * Input strings which are null are skipped. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_float(e: Column, k: Int): Column = - Column.fn("kll_merge_agg_float", e, lit(k)) + @scala.annotation.varargs + def concat_ws(sep: String, exprs: Column*): Column = + Column.fn("concat_ws", lit(sep) +: exprs: _*) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Computes the first argument into a string from a binary using the provided character set (one + * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either + * argument is null, the result will also be null. * - * @param columnName - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param value + * The target column to work on. A column that evaluates to a binary. + * @param charset + * The charset to use to decode to. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_float(columnName: String, k: Int): Column = - kll_merge_agg_float(Column(columnName), k) + def decode(value: Column, charset: String): Column = + Column.fn("decode", value, lit(charset)) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Computes the first argument into a binary from a string using the provided character set (one + * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either + * argument is null, the result will also be null. * - * @param e - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param value + * The target column to work on. A column that evaluates to a string. + * @param charset + * The charset to use to encode. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_float(e: Column): Column = - Column.fn("kll_merge_agg_float", e) + def encode(value: Column, charset: String): Column = + Column.fn("encode", value, lit(charset)) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Returns true if the input is a valid UTF-8 string, otherwise returns false. * - * @param columnName - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def kll_merge_agg_float(columnName: String): Column = - kll_merge_agg_float(Column(columnName)) + def is_valid_utf8(str: Column): Column = + Column.fn("is_valid_utf8", str) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the + * Unicode replacement character (U+FFFD). * - * @param e - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_double(e: Column, k: Column): Column = - Column.fn("kll_merge_agg_double", e, k) + def make_valid_utf8(str: Column): Column = + Column.fn("make_valid_utf8", str) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Returns the input value if it corresponds to a valid UTF-8 string, or emits a + * SparkIllegalArgumentException exception otherwise. * - * @param e - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_double(e: Column, k: Int): Column = - Column.fn("kll_merge_agg_double", e, lit(k)) + def validate_utf8(str: Column): Column = + Column.fn("validate_utf8", str) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. * - * @param columnName - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param str + * the input value. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def kll_merge_agg_double(columnName: String, k: Int): Column = - kll_merge_agg_double(Column(columnName), k) + def try_validate_utf8(str: Column): Column = + Column.fn("try_validate_utf8", str) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Returns the Unicode normalization of `str` using the given normalization `form`. Valid forms + * are 'NFC', 'NFD', 'NFKC', and 'NFKD', as defined by Unicode Standard Annex #15. The form name + * is case-insensitive. Normalization is backed by Spark's bundled ICU4J library rather than the + * JVM's own Unicode data, so results are stable across JVM vendors and versions. * - * @param e - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 - * @return - * Returns a column that evaluates to a binary. + * @param str + * the input string to normalize. + * @param form + * the normalization form: 'NFC', 'NFD', 'NFKC', or 'NFKD'. + * @group string_funcs + * @since 4.4.0 */ - def kll_merge_agg_double(e: Column): Column = - Column.fn("kll_merge_agg_double", e) + def normalize(str: Column, form: Column): Column = + Column.fn("normalize", str, form) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Returns the Unicode normalization of `str` using the default form 'NFC'. To use a different + * form, call the two-argument overload. * - * @param columnName - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 - * @return - * Returns a column that evaluates to a binary. + * @param str + * the input string to normalize. + * @group string_funcs + * @since 4.4.0 */ - def kll_merge_agg_double(columnName: String): Column = - kll_merge_agg_double(Column(columnName)) + def normalize(str: Column): Column = + Column.fn("normalize", str) /** - * Aggregate function: returns the concatenation of non-null input values. + * Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places with + * HALF_EVEN round mode, and returns the result as a string column. * - * @param e - * The target column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs - * @since 4.0.0 + * If d is 0, the result has no decimal point or fractional part. If d is less than 0, the + * result will be null. + * + * @param x + * the numeric value to be formatted. A column that evaluates to a numeric. + * @param d + * the number of decimal places. A column that evaluates to an integral. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def listagg(e: Column): Column = Column.fn("listagg", e) + def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) /** - * Aggregate function: returns the concatenation of non-null input values, separated by the - * delimiter. + * Formats the arguments in printf-style and returns the result as a string column. * - * @param e - * The target column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * The delimiter used to separate the values. A column that evaluates to a string or binary. - * Must be a constant. - * @group agg_funcs - * @since 4.0.0 + * @param format + * the format string that can contain embedded format tags. A column that evaluates to a + * string. Must be a constant. + * @param arguments + * the values to be used in formatting. Columns that evaluate to any type. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) + @scala.annotation.varargs + def format_string(format: String, arguments: Column*): Column = + Column.fn("format_string", lit(format) +: arguments: _*) /** - * Aggregate function: returns the concatenation of distinct non-null input values. + * Returns a new string column by converting the first letter of each word to uppercase. Words + * are delimited by whitespace. + * + * For example, "hello world" will become "Hello World". * * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs - * @since 4.0.0 + * the target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) + def initcap(e: Column): Column = Column.fn("initcap", e) /** - * Aggregate function: returns the concatenation of distinct non-null input values, separated by - * the delimiter. + * Locate the position of the first occurrence of substr column in the given string. Returns + * null if either of the arguments are null. * - * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * the delimiter to separate the values. A column that evaluates to a string or binary. Must - * be a constant. - * @group agg_funcs - * @since 4.0.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. Must be a constant. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an integer. */ - def listagg_distinct(e: Column, delimiter: Column): Column = - Column.fn("listagg", isDistinct = true, e, delimiter) + def instr(str: Column, substring: String): Column = instr(str, lit(substring)) /** - * Aggregate function: returns the concatenation of non-null input values. Alias for `listagg`. + * Locate the position of the first occurrence of substr column in the given string. Returns + * null if either of the arguments are null. * - * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * + * @group string_funcs * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def string_agg(e: Column): Column = Column.fn("string_agg", e) + def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) /** - * Aggregate function: returns the concatenation of non-null input values, separated by the - * delimiter. Alias for `listagg`. + * Locate the position of the first occurrence of `substring` in `str`, starting the search from + * position `start`. Returns null if either of the arguments are null. * - * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * the delimiter to separate the values. A column that evaluates to a string or binary. Must - * be a constant. - * @group agg_funcs - * @since 4.0.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) + def instr(str: Column, substring: Column, start: Int): Column = + Column.fn("instr", str, substring, lit(start)) /** - * Aggregate function: returns the concatenation of distinct non-null input values. Alias for - * `listagg`. + * Locate the position of the first occurrence of `substring` in `str`, starting the search from + * position `start`. Returns null if either of the arguments are null. * - * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs - * @since 4.0.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an integer. */ - def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) + def instr(str: Column, substring: Column, start: Column): Column = + Column.fn("instr", str, substring, start) /** - * Aggregate function: returns the concatenation of distinct non-null input values, separated by - * the delimiter. Alias for `listagg`. + * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the + * search from position `start`. Returns null if either of the arguments are null. * - * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. Must * be a constant. - * @group agg_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a string. - */ - def string_agg_distinct(e: Column, delimiter: Column): Column = - Column.fn("string_agg", isDistinct = true, e, delimiter) - - /** - * Aggregate function: alias for `var_samp`. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * @note + * The `occurrence` parameter must be a positive integer. * - * @param e - * the column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def variance(e: Column): Column = Column.fn("variance", e) + def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = + Column.fn("instr", str, substring, lit(start), lit(occurrence)) /** - * Aggregate function: alias for `var_samp`. + * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the + * search from position `start`. Returns null if either of the arguments are null. * - * @group agg_funcs - * @since 1.6.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * @note + * The `occurrence` parameter must be a positive integer. + * + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def variance(columnName: String): Column = variance(Column(columnName)) + def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = + Column.fn("instr", str, substring, start, occurrence) /** - * Aggregate function: returns the unbiased variance of the values in a group. + * Computes the character length of a given string or number of bytes of a binary string. The + * length of character strings include the trailing spaces. The length of binary strings + * includes binary zeros. * * @param e - * the column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * the target column to work on. A column that evaluates to a string or binary. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def var_samp(e: Column): Column = Column.fn("var_samp", e) + def length(e: Column): Column = Column.fn("length", e) /** - * Aggregate function: returns the unbiased variance of the values in a group. + * Computes the character length of a given string or number of bytes of a binary string. The + * length of character strings include the trailing spaces. The length of binary strings + * includes binary zeros. * - * @group agg_funcs - * @since 1.6.0 + * @param e + * the target column to work on. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def var_samp(columnName: String): Column = var_samp(Column(columnName)) + def len(e: Column): Column = Column.fn("len", e) /** - * Aggregate function: returns the population variance of the values in a group. + * Converts a string column to lower case. * * @param e - * the column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * the target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def var_pop(e: Column): Column = Column.fn("var_pop", e) + def lower(e: Column): Column = Column.fn("lower", e) /** - * Aggregate function: returns the population variance of the values in a group. - * - * @group agg_funcs - * @since 1.6.0 + * Computes the Levenshtein distance of the two given string columns if it's less than or equal + * to a given threshold. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. + * @param threshold + * the maximum distance to compute. A column that evaluates to an integral. Must be a + * constant. * @return - * Returns a column that evaluates to a double. + * result distance, or -1. Returns a column that evaluates to an integer. + * @group string_funcs + * @since 3.5.0 */ - def var_pop(columnName: String): Column = var_pop(Column(columnName)) + def levenshtein(l: Column, r: Column, threshold: Int): Column = + Column.fn("levenshtein", l, r, lit(threshold)) /** - * Aggregate function: returns the average of the independent variable for non-null pairs in a - * group, where `y` is the dependent variable and `x` is the independent variable. - * - * @param y - * the dependent variable. A column that evaluates to a numeric. - * @param x - * the independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * Computes the Levenshtein distance of the two given string columns. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) + def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) /** - * Aggregate function: returns the average of the dependent variable for non-null pairs in a - * group, where `y` is the dependent variable and `x` is the independent variable. - * - * @param y - * the dependent variable. A column that evaluates to a numeric. - * @param x - * the independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * Computes the Jaro-Winkler similarity between the two given string columns. The result is a + * double between 0.0 (no similarity) and 1.0 (identical). + * @param l + * A column that evaluates to a string. + * @param r + * A column that evaluates to a string. + * @group string_funcs + * @since 4.3.0 * @return * Returns a column that evaluates to a double. */ - def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) + def jaro_winkler_similarity(l: Column, r: Column): Column = + Column.fn("jaro_winkler_similarity", l, r) /** - * Aggregate function: returns the number of non-null number pairs in a group, where `y` is the - * dependent variable and `x` is the independent variable. + * Locate the position of the first occurrence of substr. * - * @param y - * the dependent variable. A column that evaluates to a numeric. - * @param x - * the independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an integer. */ - def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) + def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) /** - * Aggregate function: returns the intercept of the univariate linear regression line for - * non-null pairs in a group, where `y` is the dependent variable and `x` is the independent - * variable. + * Locate the position of the first occurrence of substr in a string column, after position pos. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. + * @param pos + * The starting position. A column that evaluates to an integer. + * @note + * The position is not zero based, but 1 based index. returns 0 if substr could not be found + * in str. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) + def locate(substr: String, str: Column, pos: Int): Column = + Column.fn("locate", lit(substr), str, lit(pos)) /** - * Aggregate function: returns the coefficient of determination for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. + * Left-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) + def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) /** - * Aggregate function: returns the slope of the linear regression line for non-null pairs in a - * group, where `y` is the dependent variable and `x` is the independent variable. + * Left-pad the binary column with pad to a byte length of len. If the binary column is longer + * than len, the return value is shortened to len bytes. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param str + * A column that evaluates to a binary. + * @param len + * The byte length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding bytes. A column that evaluates to a binary. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) + def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) /** - * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. + * Left-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) + def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) /** - * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. + * Trim the spaces from left end for the specified string value. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param e + * A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) + def ltrim(e: Column): Column = Column.fn("ltrim", e) /** - * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. - * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * Trim the specified character string from left end for the specified string column. + * @param e + * A column that evaluates to a string. + * @param trimString + * The trim string. A column that evaluates to a string. + * @group string_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) + def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) /** - * Aggregate function: returns some value of `e` for a group of rows. - * + * Trim the specified character string from left end for the specified string column. * @param e - * The column to return some value from. A column of any type. - * @group agg_funcs - * @since 3.5.0 + * A column that evaluates to a string. + * @param trim + * The trim string. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def any_value(e: Column): Column = Column.fn("any_value", e) + def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) /** - * Aggregate function: returns some value of `e` for a group of rows. If `ignoreNulls` is true, - * returns only non-null values. + * Calculates the byte length for the specified string column. * * @param e - * The column to return some value from. A column of any type. - * @param ignoreNulls - * If true, returns only non-null values. A column that evaluates to a boolean. Must be a - * constant. - * @group agg_funcs - * @since 3.5.0 + * A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def any_value(e: Column, ignoreNulls: Column): Column = - Column.fn("any_value", e, ignoreNulls) + def octet_length(e: Column): Column = Column.fn("octet_length", e) /** - * Aggregate function: returns the number of `TRUE` values for the expression. + * Marks a given column with specified collation. * * @param e - * The expression to count TRUE values of. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * A column that evaluates to a string. + * @param collation + * The collation name. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def count_if(e: Column): Column = Column.fn("count_if", e) + def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) /** - * Returns the current time at the start of query evaluation. Note that the result will contain - * 6 fractional digits of seconds. + * Returns the collation name of a given column. * + * @param e + * A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * A time. Returns a column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column that evaluates to a string. */ - def current_time(): Column = { - Column.fn("current_time") - } + def collation(e: Column): Column = Column.fn("collation", e) /** - * Returns the current time at the start of query evaluation. + * Returns a count of the number of times that the regular expression pattern `regexp` is + * matched in the string `str`. * - * @param precision - * An integer literal in the range [0..6], indicating how many fractional digits of seconds to - * include in the result. A column that evaluates to an integer. Must be a constant. + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * A time. Returns a column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column that evaluates to an integer. */ - def current_time(precision: Int): Column = { - Column.fn("current_time", lit(precision)) - } + def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) /** - * Aggregate function: computes a histogram on numeric 'expr' using nb bins. The return value is - * an array of (x,y) pairs representing the centers of the histogram's bins. As the value of - * 'nb' is increased, the histogram approximation gets finer-grained, but may yield artifacts - * around outliers. In practice, 20-40 histogram bins appear to work well, with more bins being - * required for skewed or smaller datasets. Note that this function creates a histogram with - * non-uniform bin widths. It offers no guarantees in terms of the mean-squared-error of the - * histogram, but in practice is comparable to the histograms produced by the R/S-Plus - * statistical computing packages. Note: the output type of the 'x' field in the return value is - * propagated from the input value consumed in the aggregate function. + * Extract a specific group matched by a Java regex, from the specified string column. If the + * regex did not match, or the specified group did not match, an empty string is returned. if + * the specified group index exceeds the group count of regex, an IllegalArgumentException will + * be thrown. * * @param e - * The column to compute the histogram on. A column that evaluates to a numeric. - * @param nBins - * The number of histogram bins. A column that evaluates to an integral. Must be a constant. - * @group agg_funcs + * target column to work on. A column that evaluates to a string. + * @param exp + * regex pattern to apply. A string. Must be a constant. + * @param groupIdx + * matched group id. An integer. Must be a constant. + * @group string_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = + Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) + + /** + * Extract all strings in the `str` that match the `regexp` expression and corresponding to the + * first regex group index. + * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return * Returns a column that evaluates to an array. */ - def histogram_numeric(e: Column, nBins: Column): Column = - Column.fn("histogram_numeric", e, nBins) + def regexp_extract_all(str: Column, regexp: Column): Column = + Column.fn("regexp_extract_all", str, regexp) /** - * Aggregate function: returns true if all values of `e` are true. + * Extract all strings in the `str` that match the `regexp` expression and corresponding to the + * regex group index. * - * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def every(e: Column): Column = Column.fn("every", e) + def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = + Column.fn("regexp_extract_all", str, regexp, idx) /** - * Aggregate function: returns true if all values of `e` are true. + * Replace all substrings of the specified string value that match regexp with rep. * * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def bool_and(e: Column): Column = Column.fn("bool_and", e) + def regexp_replace(e: Column, pattern: String, replacement: String): Column = + regexp_replace(e, lit(pattern), lit(replacement)) /** - * Aggregate function: returns true if at least one value of `e` is true. + * Replace all substrings of the specified string value that match regexp with rep, starting at + * the specified position `pos`. * * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. + * @param pos + * position to start replacement. The first position is 1. An integer. Must be a constant. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def some(e: Column): Column = Column.fn("some", e) + def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = + regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) /** - * Aggregate function: returns true if at least one value of `e` is true. + * Replace all substrings of the specified string value that match regexp with rep. * * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. + * @group string_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def any(e: Column): Column = Column.fn("any", e) + def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = + Column.fn("regexp_replace", e, pattern, replacement) /** - * Aggregate function: returns true if at least one value of `e` is true. + * Replace all substrings of the specified string value that match regexp with rep, starting at + * the specified position `pos`. * * @param e - * column to check if at least one value is true. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. + * @param pos + * position to start replacement. The first position is 1. A column that evaluates to an + * integer. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def bool_or(e: Column): Column = Column.fn("bool_or", e) + def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = + Column.fn("regexp_replace", e, pattern, replacement, pos) /** - * Aggregate function: returns the bitwise AND of all non-null input values, or null if none. + * Returns the substring that matches the regular expression `regexp` within the string `str`. + * If the regular expression is not found, the result is null. * - * @param e - * target column to compute on. A column that evaluates to an integral. - * @group agg_funcs + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def bit_and(e: Column): Column = Column.fn("bit_and", e) + def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) /** - * Aggregate function: returns the bitwise OR of all non-null input values, or null if none. + * Searches a string for a regular expression and returns an integer that indicates the + * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match + * is found, returns 0. * - * @param e - * target column to compute on. A column that evaluates to an integral. - * @group agg_funcs + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def bit_or(e: Column): Column = Column.fn("bit_or", e) + def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) /** - * Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. + * Searches a string for a regular expression and returns an integer that indicates the + * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match + * is found, returns 0. * - * @param e - * target column to compute on. A column that evaluates to an integral. - * @group agg_funcs + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def bit_xor(e: Column): Column = Column.fn("bit_xor", e) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Window functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def regexp_instr(str: Column, regexp: Column, idx: Column): Column = + Column.fn("regexp_instr", str, regexp, idx) /** - * Window function: computes the differences between consecutive cumulative counter values in a - * time series, thereby converting the counter from the cumulative to the delta format. - * - * Gracefully handles counter resets by returning NULL. Counter resets are detected when the - * counter value decreases. - * - * Use the PARTITION BY clause of the window to separate independent counters. This is done by - * specifying all columns which uniquely identify a time series. These are typically the counter - * name and any attributes tied to the counter. - * - * Use the ORDER BY clause of the window to order the observations by the associated timestamp - * in ascending order. - * - * @param value - * A cumulative counter. Must be a numeric data type. Must be non-negative. + * Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse + * of base64. * + * @param e + * target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. Returns a column of the - * same type as the input. - * @group window_funcs - * @since 4.3.0 + * Returns a column that evaluates to a binary. */ - def counter_diff(value: Column): Column = Column.fn("counter_diff", value) + def unbase64(e: Column): Column = Column.fn("unbase64", e) /** - * Window function: computes the differences between consecutive cumulative counter values in a - * time series, thereby converting the counter from the cumulative to the delta format. - * - * Gracefully handles counter resets by returning NULL. Counter resets are detected when the - * counter value decreases, or when the start time advances between rows. - * - * Use the PARTITION BY clause of the window to separate independent counters. This is done by - * specifying all columns which uniquely identify a time series. These are typically the counter - * name and any attributes tied to the counter. - * - * Use the ORDER BY clause of the window to order the observations by the associated timestamp - * in ascending order. - * - * @param value - * A cumulative counter. Must be a numeric data type. Must be non-negative. - * - * @param startTime - * A timestamp indicating when the counter was last set to zero. Used to signal counter - * resets. + * Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary column. This is + * the reverse of to_base32. * - * @return - * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. Returns a column of the - * same type as the input. - * @group window_funcs + * @param e + * target column to work on. A column that evaluates to a string. + * @group string_funcs * @since 4.3.0 + * @return + * Returns a column that evaluates to a binary. */ - def counter_diff(value: Column, startTime: Column): Column = - Column.fn("counter_diff", value, startTime) + def from_base32(e: Column): Column = Column.fn("from_base32", e) /** - * Window function: returns the cumulative distribution of values within a window partition, - * i.e. the fraction of rows that are below the current row. + * Right-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * {{{ - * N = total number of rows in the partition - * cumeDist(x) = number of values before (and including) x / N - * }}} - * - * @group window_funcs - * @since 1.6.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param len + * length of the final string. An integer. Must be a constant. + * @param pad + * chars to append. A string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def cume_dist(): Column = Column.fn("cume_dist") + def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) /** - * Window function: returns the rank of rows within a window partition, without any gaps. - * - * The difference between rank and dense_rank is that denseRank leaves no gaps in ranking - * sequence when there are ties. That is, if you were ranking a competition using dense_rank and - * had three people tie for second place, you would say that all three were in second place and - * that the next person came in third. Rank would give me sequential numbers, making the person - * that came in third place (after the ties) would register as coming in fifth. - * - * This is equivalent to the DENSE_RANK function in SQL. + * Right-pad the binary column with pad to a byte length of len. If the binary column is longer + * than len, the return value is shortened to len bytes. * - * @group window_funcs - * @since 1.6.0 + * @param str + * target column to work on. A column that evaluates to a binary. + * @param len + * byte length of the final binary. An integer. Must be a constant. + * @param pad + * bytes to append. A binary. Must be a constant. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def dense_rank(): Column = Column.fn("dense_rank") + def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) /** - * Window function: returns the value that is `offset` rows before the current row, and `null` - * if there is less than `offset` rows before the current row. For example, an `offset` of one - * will return the previous row at any given point in the window partition. - * - * This is equivalent to the LAG function in SQL. + * Right-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param len + * length of the final result. A column that evaluates to an integer. + * @param pad + * chars or bytes to append. A column that evaluates to a string or binary. + * @group string_funcs + * @since 4.0.0 * @return * Returns a column of the same type as the input. */ - def lag(e: Column, offset: Int): Column = lag(e, offset, null) + def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) /** - * Window function: returns the value that is `offset` rows before the current row, and `null` - * if there is less than `offset` rows before the current row. For example, an `offset` of one - * will return the previous row at any given point in the window partition. - * - * This is equivalent to the LAG function in SQL. + * Repeats a string column n times, and returns it as a new string column. * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. Must be a + * constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) + def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) /** - * Window function: returns the value that is `offset` rows before the current row, and - * `defaultValue` if there is less than `offset` rows before the current row. For example, an - * `offset` of one will return the previous row at any given point in the window partition. - * - * This is equivalent to the LAG function in SQL. + * Repeats a string column n times, and returns it as a new string column. * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def lag(columnName: String, offset: Int, defaultValue: Any): Column = { - lag(Column(columnName), offset, defaultValue) - } + def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) /** - * Window function: returns the value that is `offset` rows before the current row, and - * `defaultValue` if there is less than `offset` rows before the current row. For example, an - * `offset` of one will return the previous row at any given point in the window partition. - * - * This is equivalent to the LAG function in SQL. + * Trim the spaces from right end for the specified string value. * * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 + * target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def lag(e: Column, offset: Int, defaultValue: Any): Column = { - lag(e, offset, defaultValue, false) - } + def rtrim(e: Column): Column = Column.fn("rtrim", e) /** - * Window function: returns the value that is `offset` rows before the current row, and - * `defaultValue` if there is less than `offset` rows before the current row. `ignoreNulls` - * determines whether null values of row are included in or eliminated from the calculation. For - * example, an `offset` of one will return the previous row at any given point in the window - * partition. - * - * This is equivalent to the LAG function in SQL. - * + * Trim the specified character string from right end for the specified string column. * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @param ignoreNulls - * whether to ignore null values. A column that evaluates to a boolean. Must be a constant. - * @group window_funcs - * @since 3.2.0 + * target column to work on. A column that evaluates to a string. + * @param trimString + * the trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 2.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = - Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) + def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) /** - * Window function: returns the value that is `offset` rows after the current row, and `null` if - * there is less than `offset` rows after the current row. For example, an `offset` of one will - * return the next row at any given point in the window partition. - * - * This is equivalent to the LEAD function in SQL. - * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * Trim the specified character string from right end for the specified string column. + * @param e + * target column to work on. A column that evaluates to a string. + * @param trim + * the trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } + def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) /** - * Window function: returns the value that is `offset` rows after the current row, and `null` if - * there is less than `offset` rows after the current row. For example, an `offset` of one will - * return the next row at any given point in the window partition. - * - * This is equivalent to the LEAD function in SQL. + * Returns the soundex code for the specified expression. * * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } + def soundex(e: Column): Column = Column.fn("soundex", e) /** - * Window function: returns the value that is `offset` rows after the current row, and - * `defaultValue` if there is less than `offset` rows after the current row. For example, an - * `offset` of one will return the next row at any given point in the window partition. + * Splits str around matches of the given pattern. * - * This is equivalent to the LEAD function in SQL. + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a string representing a regular expression. The regex string should be a Java regular + * expression. A column that evaluates to a string. * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - def lead(columnName: String, offset: Int, defaultValue: Any): Column = { - lead(Column(columnName), offset, defaultValue) - } + def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) /** - * Window function: returns the value that is `offset` rows after the current row, and - * `defaultValue` if there is less than `offset` rows after the current row. For example, an - * `offset` of one will return the next row at any given point in the window partition. + * Splits str around matches of the given pattern. * - * This is equivalent to the LEAD function in SQL. + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a column of string representing a regular expression. The regex string should be a Java + * regular expression. A column that evaluates to a string. * - * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - def lead(e: Column, offset: Int, defaultValue: Any): Column = { - lead(e, offset, defaultValue, false) - } + def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) /** - * Window function: returns the value that is `offset` rows after the current row, and - * `defaultValue` if there is less than `offset` rows after the current row. `ignoreNulls` - * determines whether null values of row are included in or eliminated from the calculation. The - * default value of `ignoreNulls` is false. For example, an `offset` of one will return the next - * row at any given point in the window partition. + * Splits str around matches of the given pattern. * - * This is equivalent to the LEAD function in SQL. + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a string representing a regular expression. The regex string should be a Java regular + * expression. A column that evaluates to a string. + * @param limit + * an integer expression which controls the number of times the regex is applied.
    + *
  • limit greater than 0: The resulting array's length will not be more than limit, and the + * resulting array's last entry will contain all input beyond the last matched regex.
    • + *
  • limit less than or equal to 0: `regex` will be applied as many times as possible, and + * the resulting array can be of any size.
A column that evaluates to an integer. * - * @param e - * The column to compute the lead value for. A column of any type. - * @param offset - * Number of rows after the current row to look ahead. A column that evaluates to an integral. - * Must be a constant. - * @param defaultValue - * Value to return when there are fewer than `offset` rows after the current row. A column of - * any type. Must be a constant. - * @param ignoreNulls - * Whether to skip null values when computing the result. A column that evaluates to a - * boolean. Must be a constant. - * @group window_funcs - * @since 3.2.0 + * @group string_funcs + * @since 3.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = - Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) + def split(str: Column, pattern: String, limit: Int): Column = + Column.fn("split", str, lit(pattern), lit(limit)) /** - * Window function: returns the value that is the `offset`th row of the window frame (counting - * from 1), and `null` if the size of window frame is less than `offset` rows. - * - * It will return the `offset`th non-null value it sees when ignoreNulls is set to true. If all - * values are null, then null is returned. + * Splits str around matches of the given pattern. * - * This is equivalent to the nth_value function in SQL. + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a column of string representing a regular expression. The regex string should be a Java + * regular expression. A column that evaluates to a string. + * @param limit + * a column of integer expression which controls the number of times the regex is applied. + *
  • limit greater than 0: The resulting array's length will not be more than limit, + * and the resulting array's last entry will contain all input beyond the last matched + * regex.
  • limit less than or equal to 0: `regex` will be applied as many times as + * possible, and the resulting array can be of any size.
A column that evaluates to + * an integer. * - * @param e - * The column to extract the value from. A column of any type. - * @param offset - * The 1-based row number within the window frame to use as the value. A column that evaluates - * to an integral. Must be a constant. - * @param ignoreNulls - * Whether the nth value should skip nulls when determining which row to use. A column that - * evaluates to a boolean. Must be a constant. - * @group window_funcs - * @since 3.1.0 + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = - Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) + def split(str: Column, pattern: Column, limit: Column): Column = + Column.fn("split", str, pattern, limit) /** - * Window function: returns the value that is the `offset`th row of the window frame (counting - * from 1), and `null` if the size of window frame is less than `offset` rows. + * Substring starts at `pos` and is of length `len` when str is String type or returns the slice + * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * - * This is equivalent to the nth_value function in SQL. + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. Must be a constant. + * @param len + * length of chars. A column that evaluates to an integral. Must be a constant. + * @note + * The position is not zero based, but 1 based index. * - * @param e - * The column to extract the value from. A column of any type. - * @param offset - * The 1-based row number within the window frame to use as the value. A column that evaluates - * to an integral. Must be a constant. - * @group window_funcs - * @since 3.1.0 + * @group string_funcs + * @since 1.5.0 * @return * Returns a column of the same type as the input. */ - def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) + def substring(str: Column, pos: Int, len: Int): Column = + Column.fn("substring", str, lit(pos), lit(len)) /** - * Window function: returns the ntile group id (from 1 to `n` inclusive) in an ordered window - * partition. For example, if `n` is 4, the first quarter of the rows will get value 1, the - * second quarter will get 2, the third quarter will get 3, and the last quarter will get 4. + * Substring starts at `pos` and is of length `len` when str is String type or returns the slice + * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * - * This is equivalent to the NTILE function in SQL. + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. + * @param len + * length of chars. A column that evaluates to an integral. + * @note + * The position is not zero based, but 1 based index. * - * @param n - * The number of groups to divide the window partition into. A column that evaluates to an - * integral. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def ntile(n: Int): Column = Column.fn("ntile", lit(n)) + def substring(str: Column, pos: Column, len: Column): Column = + Column.fn("substring", str, pos, len) /** - * Window function: returns the relative rank (i.e. percentile) of rows within a window - * partition. - * - * This is computed by: - * {{{ - * (rank of row in its partition - 1) / (number of rows in the partition - 1) - * }}} - * - * This is equivalent to the PERCENT_RANK function in SQL. + * Returns the substring from string str before count occurrences of the delimiter delim. If + * count is positive, everything the left of the final delimiter (counting from left) is + * returned. If count is negative, every to the right of the final delimiter (counting from the + * right) is returned. substring_index performs a case-sensitive match when searching for delim. * - * @group window_funcs - * @since 1.6.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param delim + * delimiter of values. A column that evaluates to a string. Must be a constant. + * @param count + * number of occurrences. A column that evaluates to an integral. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def percent_rank(): Column = Column.fn("percent_rank") + def substring_index(str: Column, delim: String, count: Int): Column = + Column.fn("substring_index", str, lit(delim), lit(count)) /** - * Window function: returns the rank of rows within a window partition. - * - * The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking - * sequence when there are ties. That is, if you were ranking a competition using dense_rank and - * had three people tie for second place, you would say that all three were in second place and - * that the next person came in third. Rank would give me sequential numbers, making the person - * that came in third place (after the ties) would register as coming in fifth. - * - * This is equivalent to the RANK function in SQL. + * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of + * `src` and proceeding for `len` bytes. * - * @group window_funcs - * @since 1.4.0 + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. + * @param len + * the number of bytes to replace in src. A column that evaluates to an integral. + * @group string_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def rank(): Column = Column.fn("rank") + def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = + Column.fn("overlay", src, replace, pos, len) /** - * Window function: returns a sequential number starting at 1 within a window partition. + * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of + * `src`. * - * @group window_funcs - * @since 1.6.0 + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. + * @group string_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a string. */ - def row_number(): Column = Column.fn("row_number") + def overlay(src: Column, replace: Column, pos: Column): Column = + Column.fn("overlay", src, replace, pos) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Non-aggregate functions - ////////////////////////////////////////////////////////////////////////////////////////////// + /** + * Splits a string into arrays of sentences, where each sentence is an array of words. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. + * @param country + * a country of the locale. A column that evaluates to a string. + * @group string_funcs + * @since 3.2.0 + * @return + * Returns a column that evaluates to an array. + */ + def sentences(string: Column, language: Column, country: Column): Column = + Column.fn("sentences", string, language, country) /** - * Creates a new array column. The input columns must all have the same data type. - * - * @param cols - * The columns to combine into an array. Each is a column of any type, and all must share the - * same data type. - * @group array_funcs - * @since 1.4.0 + * Splits a string into arrays of sentences, where each sentence is an array of words. The + * default `country`('') is used. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return * Returns a column that evaluates to an array. */ - @scala.annotation.varargs - def array(cols: Column*): Column = Column.fn("array", cols: _*) + def sentences(string: Column, language: Column): Column = + Column.fn("sentences", string, language) /** - * Creates a new array column. The input columns must all have the same data type. - * - * @group array_funcs - * @since 1.4.0 + * Splits a string into arrays of sentences, where each sentence is an array of words. The + * default locale is used. + * @param string + * a string to be split. A column that evaluates to a string. + * @group string_funcs + * @since 3.2.0 * @return * Returns a column that evaluates to an array. */ - @scala.annotation.varargs - def array(colName: String, colNames: String*): Column = { - array((colName +: colNames).map(col): _*) - } + def sentences(string: Column): Column = Column.fn("sentences", string) /** - * Creates a new map column. The input columns must be grouped as key-value pairs, e.g. (key1, - * value1, key2, value2, ...). The key columns must all have the same data type, and can't be - * null. The value columns must all have the same data type. + * Translate any character in the src by a character in replaceString. The characters in + * replaceString correspond to the characters in matchingString. The translate will happen when + * any character in the string matches the character in the `matchingString`. * - * @param cols - * The columns grouped as key-value pairs (key1, value1, key2, value2, ...). Each is a column - * of any type; key columns must share a type and value columns must share a type. - * @group map_funcs - * @since 2.0 + * @param src + * source column to work on. A column that evaluates to a string. + * @param matchingString + * matching characters. A column that evaluates to a string. Must be a constant. + * @param replaceString + * characters for replacement. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def map(cols: Column*): Column = Column.fn("map", cols: _*) + def translate(src: Column, matchingString: String, replaceString: String): Column = + Column.fn("translate", src, lit(matchingString), lit(replaceString)) /** - * Creates a struct with the given field names and values. + * Trim the spaces from both ends for the specified string column. * - * @param cols - * The field names and values grouped as pairs (name1, value1, name2, value2, ...). Names are - * columns that evaluate to a string; values are columns of any type. - * @group struct_funcs - * @since 3.5.0 + * @param e + * The string column to trim. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) + def trim(e: Column): Column = Column.fn("trim", e) /** - * Creates a new map column. The array in the first column is used for keys. The array in the - * second column is used for values. All elements in the array for key should not be null. - * - * @param keys - * The array of keys for the map; elements must not be null. A column that evaluates to an - * array. - * @param values - * The array of values for the map. A column that evaluates to an array. - * @group map_funcs - * @since 2.4 + * Trim the specified character from both ends for the specified string column. + * @param e + * The string column to trim. A column that evaluates to a string. + * @param trimString + * The trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a string. */ - def map_from_arrays(keys: Column, values: Column): Column = - Column.fn("map_from_arrays", keys, values) + def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) /** - * Creates a map after splitting the text into key/value pairs using delimiters. Both - * `pairDelim` and `keyValueDelim` are treated as regular expressions. - * - * @param text - * The text to split into key/value pairs. A column that evaluates to a string. - * @param pairDelim - * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to - * a string. - * @param keyValueDelim - * Delimiter used to split key and value, treated as a regular expression. A column that - * evaluates to a string. - * @group map_funcs - * @since 3.5.0 + * Trim the specified character from both ends for the specified string column. + * @param e + * The string column to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a string. */ - def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = - Column.fn("str_to_map", text, pairDelim, keyValueDelim) + def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) /** - * Creates a map after splitting the text into key/value pairs using delimiters. The `pairDelim` - * is treated as regular expressions. + * Converts a string column to upper case. * - * @param text - * The text to split into key/value pairs. A column that evaluates to a string. - * @param pairDelim - * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to - * a string. - * @group map_funcs - * @since 3.5.0 + * @param e + * The input column to convert to upper case. A column that evaluates to a string. + * @group string_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a string. */ - def str_to_map(text: Column, pairDelim: Column): Column = - Column.fn("str_to_map", text, pairDelim) + def upper(e: Column): Column = Column.fn("upper", e) /** - * Creates a map after splitting the text into key/value pairs using delimiters. + * Converts the input `e` to a binary value based on the supplied `format`. The `format` can be + * a case-insensitive string literal of "hex", "utf-8", "utf8", or "base64". By default, the + * binary format for conversion is "hex" if `format` is omitted. The function returns NULL if at + * least one of the input parameters is NULL. * - * @param text - * The text to split into key/value pairs. A column that evaluates to a string. - * @group map_funcs + * @param e + * The input value to convert. A column that evaluates to a string. + * @param f + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a binary. */ - def str_to_map(text: Column): Column = Column.fn("str_to_map", text) + def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) /** - * Marks a DataFrame as small enough for use in broadcast joins. - * - * The following example marks the right DataFrame for broadcast hash join using `joinKey`. - * {{{ - * // left and right are DataFrames - * left.join(broadcast(right), "joinKey") - * }}} + * Converts the input `e` to a binary value based on the default format "hex". The function + * returns NULL if at least one of the input parameters is NULL. * - * @group normal_funcs - * @since 1.5.0 + * @param e + * The input value to convert. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def broadcast[U](df: Dataset[U]): df.type = { - df.hint("broadcast").asInstanceOf[df.type] - } + def to_binary(e: Column): Column = Column.fn("to_binary", e) + // scalastyle:off line.size.limit /** - * Returns the first column that is not null, or null if all inputs are null. + * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. + * The format can consist of the following characters, case insensitive: '0' or '9': Specifies + * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a + * sequence of digits in the input value, generating a result string of the same length as the + * corresponding sequence in the format string. The result string is left-padded with zeros if + * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts + * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': + * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': + * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to + * the left and right of each grouping separator. '$': Specifies the location of the $ currency + * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' + * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note + * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the + * end of the format string; specifies that the result string will be wrapped by angle brackets + * if the input value is negative. * - * For example, `coalesce(a, b, c)` will return a if a is not null, or b if a is null and b is - * not null, or c if both a and b are null but c is not null. + * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime + * Patterns. If `e` is a binary, it is converted to a string in one of the formats: + * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input + * binary is decoded to UTF-8 string. * * @param e - * the columns to work on. A column that evaluates to any type. - * @group conditional_funcs - * @since 1.3.0 + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant when `e` is a numeric or binary value. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) + // scalastyle:on line.size.limit + def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) + // scalastyle:off line.size.limit /** - * Creates a string column for the file name of the current Spark task. + * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. + * The format can consist of the following characters, case insensitive: '0' or '9': Specifies + * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a + * sequence of digits in the input value, generating a result string of the same length as the + * corresponding sequence in the format string. The result string is left-padded with zeros if + * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts + * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': + * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': + * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to + * the left and right of each grouping separator. '$': Specifies the location of the $ currency + * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' + * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note + * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the + * end of the format string; specifies that the result string will be wrapped by angle brackets + * if the input value is negative. * - * @group misc_funcs - * @since 1.6.0 + * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime + * Patterns. If `e` is a binary, it is converted to a string in one of the formats: + * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input + * binary is decoded to UTF-8 string. + * + * @param e + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant when `e` is a numeric or binary value. + * @group string_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to a string. */ - def input_file_name(): Column = Column.fn("input_file_name") + // scalastyle:on line.size.limit + def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) /** - * Return true iff the column is NaN. + * Convert string 'e' to a number based on the string format 'format'. Throws an exception if + * the conversion fails. The format can consist of the following characters, case insensitive: + * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format + * string matches a sequence of digits in the input string. If the 0/9 sequence starts with 0 + * and is before the decimal point, it can only match a digit sequence of the same size. + * Otherwise, if the sequence starts with 9 or is after the decimal point, it can match a digit + * sequence that has the same or smaller size. '.' or 'D': Specifies the position of the decimal + * point (optional, only allowed once). ',' or 'G': Specifies the position of the grouping + * (thousands) separator (,). There must be a 0 or 9 to the left and right of each grouping + * separator. 'expr' must match the grouping separator relevant for the size of the number. '$': + * Specifies the location of the $ currency sign. This character may only be specified once. 'S' + * or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at the + * beginning or end of the format string). Note that 'S' allows '-' but 'MI' does not. 'PR': + * Only allowed at the end of the format string; specifies that 'expr' indicates a negative + * number with wrapping angled brackets. * * @param e - * the column to check. A column that evaluates to a numeric. - * @group predicate_funcs - * @since 1.6.0 + * The input string to convert to a number. A column that evaluates to a string. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a decimal. */ - def isnan(e: Column): Column = e.isNaN + def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) /** - * Return true iff the column is null. + * Replaces all occurrences of `search` with `replace`. * - * @param e - * the column to check. A column that evaluates to any type. - * @group predicate_funcs - * @since 1.6.0 - * @return - * Returns a column that evaluates to a boolean. + * @param src + * A column of strings to be replaced. A column that evaluates to a string. + * @param search + * A column of strings. If `search` is not found in `str`, `str` is returned unchanged. A + * column that evaluates to a string. + * @param replace + * A column of strings. If `replace` is not specified or is an empty string, nothing replaces + * the string that is removed from `str`. A column that evaluates to a string. + * + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ - def isnull(e: Column): Column = e.isNull + def replace(src: Column, search: Column, replace: Column): Column = + Column.fn("replace", src, search, replace) /** - * A column expression that generates monotonically increasing 64-bit integers. - * - * The generated ID is guaranteed to be monotonically increasing and unique, but not - * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the - * record number within each partition in the lower 33 bits. The assumption is that the data - * frame has less than 1 billion partitions, and each partition has less than 8 billion records. - * - * As an example, consider a `DataFrame` with two partitions, each with 3 records. This - * expression would return the following IDs: + * Replaces all occurrences of `search` with `replace`. * - * {{{ - * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. - * }}} + * @param src + * A column of strings to be replaced. A column that evaluates to a string. + * @param search + * A column of strings. If `search` is not found in `src`, `src` is returned unchanged. A + * column that evaluates to a string. * - * @group misc_funcs - * @since 1.4.0 + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - @deprecated("Use monotonically_increasing_id()", "2.0.0") - def monotonicallyIncreasingId(): Column = monotonically_increasing_id() + def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) /** - * A column expression that generates monotonically increasing 64-bit integers. - * - * The generated ID is guaranteed to be monotonically increasing and unique, but not - * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the - * record number within each partition in the lower 33 bits. The assumption is that the data - * frame has less than 1 billion partitions, and each partition has less than 8 billion records. - * - * As an example, consider a `DataFrame` with two partitions, each with 3 records. This - * expression would return the following IDs: - * - * {{{ - * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. - * }}} + * Splits `str` by delimiter and return requested part of the split (1-based). If any input is + * null, returns null. if `partNum` is out of range of split parts, returns empty string. If + * `partNum` is 0, throws an error. If `partNum` is negative, the parts are counted backward + * from the end of the string. If the `delimiter` is an empty string, the `str` is not split. * - * @group misc_funcs - * @since 1.6.0 + * @param str + * A column of strings to be split. A column that evaluates to a string. + * @param delimiter + * The delimiter used for split. A column that evaluates to a string. + * @param partNum + * The requested part of the split (1-based). A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") + def split_part(str: Column, delimiter: Column, partNum: Column): Column = + Column.fn("split_part", str, delimiter, partNum) /** - * Returns col1 if it is not NaN, or col2 if col1 is NaN. - * - * Both inputs should be floating point columns (DoubleType or FloatType). + * Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of + * byte array that starts at `pos` and is of length `len`. * - * @param col1 - * the first column to check. A column that evaluates to a numeric. - * @param col2 - * the column to return if the first is NaN. A column that evaluates to a numeric. - * @group conditional_funcs - * @since 1.5.0 + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. + * @param len + * The length of the substring. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the first input. + * Returns a column of the same type as the input. */ - def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) + def substr(str: Column, pos: Column, len: Column): Column = + Column.fn("substr", str, pos, len) /** - * Unary minus, i.e. negate the expression. - * {{{ - * // Select the amount column and negates all values. - * // Scala: - * df.select( -df("amount") ) - * - * // Java: - * df.select( negate(df.col("amount")) ); - * }}} + * Returns the substring of `str` that starts at `pos`, or the slice of byte array that starts + * at `pos`. * - * @param e - * the column to negate. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 1.3.0 + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - def negate(e: Column): Column = -e + def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) /** - * Inversion of boolean expression, i.e. NOT. - * {{{ - * // Scala: select rows that are not active (isActive === false) - * df.filter( !df("isActive") ) - * - * // Java: - * df.filter( not(df.col("isActive")) ); - * }}} + * Formats the arguments in printf-style and returns the result as a string column. * - * @param e - * the column to invert. A column that evaluates to a boolean. - * @group predicate_funcs - * @since 1.3.0 + * @param format + * A format string that can contain embedded format tags. A column that evaluates to a string. + * @param arguments + * The values to be used in formatting. Columns that evaluate to any type. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def not(e: Column): Column = !e + @scala.annotation.varargs + def printf(format: Column, arguments: Column*): Column = + Column.fn("printf", (format +: arguments): _*) /** - * Generate a random column with independent and identically distributed (i.i.d.) samples - * uniformly distributed in [0.0, 1.0). - * - * @param seed - * the seed for the random generator. - * @note - * The function is non-deterministic in general case. + * Returns the position of the first occurrence of `substr` in `str` after position `start`. The + * given `start` and return value are 1-based. * - * @group math_funcs - * @since 1.4.0 + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. + * @param start + * The 1-based position to start the search from. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def rand(seed: Long): Column = Column.fn("rand", lit(seed)) + def position(substr: Column, str: Column, start: Column): Column = + Column.fn("position", substr, str, start) /** - * Generate a random column with independent and identically distributed (i.i.d.) samples - * uniformly distributed in [0.0, 1.0). - * - * @note - * The function is non-deterministic in general case. + * Returns the position of the first occurrence of `substr` in `str` after position `1`. The + * return value are 1-based. * - * @group math_funcs - * @since 1.4.0 + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def rand(): Column = rand(SparkClassUtils.random.nextLong) + def position(substr: Column, str: Column): Column = + Column.fn("position", substr, str) /** - * Generate a column with independent and identically distributed (i.i.d.) samples from the - * standard normal distribution. - * - * @param seed - * the seed for the random generator. - * @note - * The function is non-deterministic in general case. + * Returns a boolean. The value is True if str ends with suffix. Returns NULL if either input + * expression is NULL. Otherwise, returns False. Both str or suffix must be of STRING or BINARY + * type. * - * @group math_funcs - * @since 1.4.0 + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param suffix + * The suffix to test for. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a boolean. */ - def randn(seed: Long): Column = Column.fn("randn", lit(seed)) + def endswith(str: Column, suffix: Column): Column = + Column.fn("endswith", str, suffix) /** - * Generate a column with independent and identically distributed (i.i.d.) samples from the - * standard normal distribution. - * - * @note - * The function is non-deterministic in general case. + * Returns a boolean. The value is True if str starts with prefix. Returns NULL if either input + * expression is NULL. Otherwise, returns False. Both str or prefix must be of STRING or BINARY + * type. * - * @group math_funcs - * @since 1.4.0 + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param prefix + * The prefix to test for. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a boolean. */ - def randn(): Column = randn(SparkClassUtils.random.nextLong) + def startswith(str: Column, prefix: Column): Column = + Column.fn("startswith", str, prefix) /** - * Returns a string of the specified length whose characters are chosen uniformly at random from - * the following pool of characters: 0-9, a-z, A-Z. The string length must be a constant - * two-byte or four-byte integer (SMALLINT or INT, respectively). + * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the + * result is equivalent to char(n % 256) * - * @param length - * the number of characters in the string to generate. A column that evaluates to an integral. - * Must be a constant. + * @param n + * The code point value. A column that evaluates to an integral. * @group string_funcs - * @since 4.0.0 + * @since 3.5.0 * @return * Returns a column that evaluates to a string. */ - def randstr(length: Column): Column = - randstr(length, lit(SparkClassUtils.random.nextLong)) + def char(n: Column): Column = Column.fn("char", n) /** - * Returns a string of the specified length whose characters are chosen uniformly at random from - * the following pool of characters: 0-9, a-z, A-Z, with the chosen random seed. The string - * length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). + * Removes the leading and trailing space characters from `str`. * - * @param length - * the number of characters in the string to generate. A column that evaluates to an integral. - * Must be a constant. - * @param seed - * the random seed to use. A column that evaluates to an integral. + * @param str + * The string to trim. A column that evaluates to a string. * @group string_funcs - * @since 4.0.0 + * @since 3.5.0 * @return * Returns a column that evaluates to a string. */ - def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) + def btrim(str: Column): Column = Column.fn("btrim", str) /** - * Partition ID. - * - * @note - * This is non-deterministic because it depends on data partitioning and task scheduling. + * Remove the leading and trailing `trim` characters from `str`. * - * @group misc_funcs - * @since 1.6.0 + * @param str + * The string to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a string. */ - def spark_partition_id(): Column = Column.fn("spark_partition_id") + def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) /** - * Computes the square root of the specified float value. + * This is a special version of `to_binary` that performs the same operation, but returns a NULL + * value instead of raising an error if the conversion cannot be performed. * * @param e - * the value to compute the square root of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.3.0 - * @return - * Returns a column that evaluates to a double. - */ - def sqrt(e: Column): Column = Column.fn("sqrt", e) - - /** - * Computes the square root of the specified float value. - * - * @param colName - * the name of a numeric column to compute the square root of. - * @group math_funcs - * @since 1.5.0 + * The string to convert. A column that evaluates to a string. + * @param f + * The format to use for the conversion. A column that evaluates to a string. Must be a + * constant. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def sqrt(colName: String): Column = sqrt(Column(colName)) + def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) /** - * Returns the sum of `left` and `right` and the result is null on overflow. The acceptable - * input types are the same with the `+` operator. + * This is a special version of `to_binary` that performs the same operation, but returns a NULL + * value instead of raising an error if the conversion cannot be performed. * - * @param left - * the left operand. A column that evaluates to a numeric or interval. - * @param right - * the right operand. A column that evaluates to a numeric or interval. - * @group math_funcs + * @param e + * The string to convert. A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) + def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) /** - * Returns the mean calculated from values of a group and the result is null on overflow. + * Convert string `e` to a number based on the string format `format`. Returns NULL if the + * string `e` does not match the expected format. The format follows the same semantics as the + * to_number function. * * @param e - * the value to compute the mean of. A column that evaluates to a numeric or interval. - * @group agg_funcs + * The string to convert. A column that evaluates to a string. + * @param format + * The format used to convert the string to a number. A column that evaluates to a string. + * Must be a constant. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a decimal. */ - def try_avg(e: Column): Column = Column.fn("try_avg", e) + def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) /** - * Returns `dividend``/``divisor`. It always performs floating point division. Its result is - * always null if `divisor` is 0. + * Returns the character length of string data or number of bytes of binary data. The length of + * string data includes the trailing spaces. The length of binary data includes binary zeros. * - * @param left - * the dividend. A column that evaluates to a numeric or interval. - * @param right - * the divisor. A column that evaluates to a numeric. - * @group math_funcs + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) + def char_length(str: Column): Column = Column.fn("char_length", str) /** - * Returns the remainder of `dividend``/``divisor`. Its result is always null if `divisor` is 0. + * Returns the character length of string data or number of bytes of binary data. The length of + * string data includes the trailing spaces. The length of binary data includes binary zeros. * - * @param left - * the dividend. A column that evaluates to a numeric. - * @param right - * the divisor. A column that evaluates to a numeric. - * @group math_funcs - * @since 4.0.0 + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) + def character_length(str: Column): Column = Column.fn("character_length", str) /** - * Returns `left``*``right` and the result is null on overflow. The acceptable input types are - * the same with the `*` operator. + * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the + * result is equivalent to chr(n % 256) * - * @param left - * the multiplicand. A column that evaluates to a numeric or interval. - * @param right - * the multiplier. A column that evaluates to a numeric or interval. - * @group math_funcs + * @param n + * The code point. A column that evaluates to an integral. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) + def chr(n: Column): Column = Column.fn("chr", n) /** - * Returns `left``-``right` and the result is null on overflow. The acceptable input types are - * the same with the `-` operator. + * Returns a boolean. The value is True if right is found inside left. Returns NULL if either + * input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or + * BINARY type. * * @param left - * the left operand. A column that evaluates to a numeric or interval. + * The input to check, may be NULL. A column that evaluates to a string or binary. * @param right - * the right operand. A column that evaluates to a numeric or interval. - * @group math_funcs + * The input to find, may be NULL. A column that evaluates to a string or binary. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) + def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) /** - * Returns the sum calculated from values of a group and the result is null on overflow. + * Returns the `n`-th input, e.g., returns `input2` when `n` is 2. The function returns NULL if + * the index exceeds the length of the array and `spark.sql.ansi.enabled` is set to false. If + * `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException for invalid + * indices. * - * @param e - * the value to compute the sum of. A column that evaluates to a numeric or interval. - * @group agg_funcs + * @param inputs + * The index followed by the inputs to select from. Columns where the first evaluates to an + * integral and the rest evaluate to strings or binaries. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a numeric. + * Returns a column that evaluates to a string. */ - def try_sum(e: Column): Column = Column.fn("try_sum", e) + @scala.annotation.varargs + def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) /** - * Creates a new struct column. If the input column is a column in a `DataFrame`, or a derived - * column expression that is named (i.e. aliased), its name would be retained as the - * StructField's name, otherwise, the newly generated StructField's name would be auto generated - * as `col` with a suffix `index + 1`, i.e. col1, col2, col3, ... + * Returns the index (1-based) of the given string (`str`) in the comma-delimited list + * (`strArray`). Returns 0, if the string was not found or if the given string (`str`) contains + * a comma. * - * @param cols - * the columns to contain in the output struct. A column of any type. - * @group struct_funcs - * @since 1.4.0 + * @param str + * The given string to be found. A column that evaluates to a string. + * @param strArray + * The comma-delimited list. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to an integer. */ - @scala.annotation.varargs - def struct(cols: Column*): Column = Column.fn("struct", cols: _*) + def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) /** - * Creates a new struct column that composes multiple input columns. + * Returns `str` with all characters changed to lowercase. * - * @param colName - * the name of the first column to contain in the output struct. - * @param colNames - * the names of the remaining columns to contain in the output struct. - * @group struct_funcs - * @since 1.4.0 + * @param str + * A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def struct(colName: String, colNames: String*): Column = { - struct((colName +: colNames).map(col): _*) - } + def lcase(str: Column): Column = Column.fn("lcase", str) /** - * Evaluates a list of conditions and returns one of multiple possible result expressions. If - * otherwise is not defined at the end, null is returned for unmatched conditions. - * - * {{{ - * // Example: encoding gender string column into integer. - * - * // Scala: - * people.select(when(people("gender") === "male", 0) - * .when(people("gender") === "female", 1) - * .otherwise(2)) - * - * // Java: - * people.select(when(col("gender").equalTo("male"), 0) - * .when(col("gender").equalTo("female"), 1) - * .otherwise(2)) - * }}} + * Returns `str` with all characters changed to uppercase. * - * @param condition - * the condition to evaluate. A column that evaluates to a boolean. - * @param value - * the value to return when the condition is true. A literal value, or a column expression. - * @group conditional_funcs - * @since 1.4.0 + * @param str + * A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def when(condition: Column, value: Any): Column = - Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) + def ucase(str: Column): Column = Column.fn("ucase", str) /** - * Computes bitwise NOT (~) of a number. + * Returns the leftmost `len`(`len` can be string type) characters from the string `str`, if + * `len` is less or equal than 0 the result is an empty string. * - * @group bitwise_funcs - * @since 1.4.0 + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @param len + * The number of leftmost characters. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - @deprecated("Use bitwise_not", "3.2.0") - def bitwiseNOT(e: Column): Column = bitwise_not(e) + def left(str: Column, len: Column): Column = Column.fn("left", str, len) /** - * Computes bitwise NOT (~) of a number. + * Returns the rightmost `len`(`len` can be string type) characters from the string `str`, if + * `len` is less or equal than 0 the result is an empty string. + * + * @param str + * Input column or strings. A column that evaluates to a string. + * @param len + * The number of rightmost characters. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def right(str: Column, len: Column): Column = Column.fn("right", str, len) + + /** + * Returns `str` enclosed by single quotes and each instance of single quote in it is preceded + * by a backslash. + * + * @param str + * A column that evaluates to a string. + * @group string_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. + */ + def quote(str: Column): Column = Column.fn("quote", str) + + /** + * Masks the given string value. The function replaces characters with 'X' or 'x', and numbers + * with 'n'. This can be useful for creating copies of tables with sensitive information + * removed. + * + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def mask(input: Column): Column = Column.fn("mask", input) + + /** + * Masks the given string value. The function replaces upper-case characters with specific + * character, lower-case characters with 'x', and numbers with 'n'. This can be useful for + * creating copies of tables with sensitive information removed. + * + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def mask(input: Column, upperChar: Column): Column = + Column.fn("mask", input, upperChar) + + /** + * Masks the given string value. The function replaces upper-case and lower-case characters with + * the characters specified respectively, and numbers with 'n'. This can be useful for creating + * copies of tables with sensitive information removed. + * + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param lowerChar + * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def mask(input: Column, upperChar: Column, lowerChar: Column): Column = + Column.fn("mask", input, upperChar, lowerChar) + + /** + * Masks the given string value. The function replaces upper-case, lower-case characters and + * numbers with the characters specified respectively. This can be useful for creating copies of + * tables with sensitive information removed. + * + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param lowerChar + * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param digitChar + * character to replace digit characters with. Specify NULL to retain original character. A + * column that evaluates to a string. + * + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column): Column = + Column.fn("mask", input, upperChar, lowerChar, digitChar) + + /** + * Masks the given string value. This can be useful for creating copies of tables with sensitive + * information removed. + * + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param lowerChar + * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param digitChar + * character to replace digit characters with. Specify NULL to retain original character. A + * column that evaluates to a string. + * @param otherChar + * character to replace all other characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * + * @group string_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def mask( + input: Column, + upperChar: Column, + lowerChar: Column, + digitChar: Column, + otherChar: Column): Column = + Column.fn("mask", input, upperChar, lowerChar, digitChar, otherChar) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Bitwise Functions + ////////////////////////////////////////////////////////////////////////////////////////////// + + /** + * Computes bitwise NOT (~) of a number. + * + * @group bitwise_funcs + * @since 1.4.0 + * @return + * Returns a column of the same type as the input. + */ + @deprecated("Use bitwise_not", "3.2.0") + def bitwiseNOT(e: Column): Column = bitwise_not(e) + + /** + * Computes bitwise NOT (~) of a number. * * @param e * the target column to compute on. A column that evaluates to an integral. @@ -4899,2226 +4431,2865 @@ object functions { def getbit(e: Column, pos: Column): Column = Column.fn("getbit", e, pos) /** - * Parses the expression string into the column that it represents, similar to - * [[Dataset#selectExpr]]. - * {{{ - * // get the number of words of each length - * df.groupBy(expr("length(word)")).count() - * }}} + * Shift the given value numBits left. If the given value is a long value, this function will + * return a long value else it will return an integer value. * - * @group normal_funcs + * @group bitwise_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ - def expr(expr: String): Column = Column(internal.SqlExpression(expr)) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Math Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + @deprecated("Use shiftleft", "3.2.0") + def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) /** - * Computes the absolute value of a numeric value. + * Shift the given value numBits left. If the given value is a long value, this function will + * return a long value else it will return an integer value. * * @param e - * the value to compute the absolute value of. A column that evaluates to a numeric or - * interval. - * @group math_funcs - * @since 1.3.0 + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift left. A column that evaluates to an integral. Must be a + * constant. + * @group bitwise_funcs + * @since 3.2.0 * @return * Returns a column of the same type as the input. */ - def abs(e: Column): Column = Column.fn("abs", e) + def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) /** - * @param e - * the value to compute the inverse cosine of. A column that evaluates to a double. - * @return - * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column - * that evaluates to a double. + * (Signed) shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. * - * @group math_funcs - * @since 1.4.0 - */ - def acos(e: Column): Column = Column.fn("acos", e) - - /** - * @param columnName - * the value to compute the inverse cosine of. + * @group bitwise_funcs + * @since 1.5.0 * @return - * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column - * that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column of the same type as the input. */ - def acos(columnName: String): Column = acos(Column(columnName)) + @deprecated("Use shiftright", "3.2.0") + def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) /** + * (Signed) shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. + * * @param e - * the value to compute the inverse hyperbolic cosine of. A column that evaluates to a double. + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. + * @group bitwise_funcs + * @since 3.2.0 * @return - * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. - * - * @group math_funcs - * @since 3.1.0 + * Returns a column of the same type as the input. */ - def acosh(e: Column): Column = Column.fn("acosh", e) + def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) /** - * @param columnName - * the value to compute the inverse hyperbolic cosine of. + * Unsigned shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. + * + * @group bitwise_funcs + * @since 1.5.0 * @return - * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 3.1.0 + * Returns a column of the same type as the input. */ - def acosh(columnName: String): Column = acosh(Column(columnName)) + @deprecated("Use shiftrightunsigned", "3.2.0") + def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) /** - * @param e - * the value to compute the inverse sine of. A column that evaluates to a double. - * @return - * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column - * that evaluates to a double. + * Unsigned shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. * - * @group math_funcs - * @since 1.4.0 + * @param e + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. + * @group bitwise_funcs + * @since 3.2.0 + * @return + * Returns a column of the same type as the input. */ - def asin(e: Column): Column = Column.fn("asin", e) + def shiftrightunsigned(e: Column, numBits: Int): Column = + Column.fn("shiftrightunsigned", e, lit(numBits)) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Date and Timestamp Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * @param columnName - * the value to compute the inverse sine of. + * Create time from hour, minute and second fields. For invalid inputs it will throw an error. + * + * @param hour + * the hour to represent, from 0 to 23. A column that evaluates to an integer. + * @param minute + * the minute to represent, from 0 to 59. A column that evaluates to an integer. + * @param second + * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. + * @group datetime_funcs + * @since 4.1.0 * @return - * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column - * that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a time. */ - def asin(columnName: String): Column = asin(Column(columnName)) + def make_time(hour: Column, minute: Column, second: Column): Column = { + Column.fn("make_time", hour, minute, second) + } /** - * @param e - * the value to compute the inverse hyperbolic sine of. A column that evaluates to a double. - * @return - * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. + * Returns the current time at the start of query evaluation. Note that the result will contain + * 6 fractional digits of seconds. * - * @group math_funcs - * @since 3.1.0 + * @return + * A time. Returns a column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 */ - def asinh(e: Column): Column = Column.fn("asinh", e) + def current_time(): Column = { + Column.fn("current_time") + } /** - * @param columnName - * the value to compute the inverse hyperbolic sine of. + * Returns the current time at the start of query evaluation. + * + * @param precision + * An integer literal in the range [0..6], indicating how many fractional digits of seconds to + * include in the result. A column that evaluates to an integer. Must be a constant. * @return - * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 3.1.0 + * A time. Returns a column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 */ - def asinh(columnName: String): Column = asinh(Column(columnName)) + def current_time(precision: Int): Column = { + Column.fn("current_time", lit(precision)) + } /** - * @param e - * the value to compute the inverse tangent of. A column that evaluates to a double. - * @return - * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that - * evaluates to a double. + * Returns the date that is `numMonths` after `startDate`. * - * @group math_funcs - * @since 1.4.0 + * @param startDate + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param numMonths + * The number of months to add to `startDate`, can be negative to subtract months. A column + * that evaluates to an integer. + * @return + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def atan(e: Column): Column = Column.fn("atan", e) + def add_months(startDate: Column, numMonths: Int): Column = + add_months(startDate, lit(numMonths)) /** - * @param columnName - * the value to compute the inverse tangent of. + * Returns the date that is `numMonths` after `startDate`. + * + * @param startDate + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param numMonths + * A column of the number of months to add to `startDate`, can be negative to subtract months. + * A column that evaluates to an integer. * @return - * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column - * that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. + * @group datetime_funcs + * @since 3.0.0 */ - def atan(columnName: String): Column = atan(Column(columnName)) + def add_months(startDate: Column, numMonths: Column): Column = + Column.fn("add_months", startDate, numMonths) /** - * @param y - * coordinate on y-axis. A column that evaluates to a double. - * @param x - * coordinate on x-axis. A column that evaluates to a double. - * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * Returns the current date at the start of query evaluation as a date column. All calls of + * current_date within the same query return the same value. * - * @group math_funcs - * @since 1.4.0 + * @group datetime_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a date. */ - def atan2(y: Column, x: Column): Column = Column.fn("atan2", y, x) + def curdate(): Column = Column.fn("curdate") /** - * @param y - * coordinate on y-axis - * @param xName - * coordinate on x-axis + * Returns the current date at the start of query evaluation as a date column. All calls of + * current_date within the same query return the same value. + * + * @group datetime_funcs + * @since 1.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a date. */ - def atan2(y: Column, xName: String): Column = atan2(y, Column(xName)) + def current_date(): Column = Column.fn("current_date") /** - * @param yName - * coordinate on y-axis - * @param x - * coordinate on x-axis + * Returns the current session local timezone. + * + * @group datetime_funcs + * @since 3.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a string. */ - def atan2(yName: String, x: Column): Column = atan2(Column(yName), x) + def current_timezone(): Column = Column.fn("current_timezone") /** - * @param yName - * coordinate on y-axis - * @param xName - * coordinate on x-axis + * Returns the current timestamp at the start of query evaluation as a timestamp column. All + * calls of current_timestamp within the same query return the same value. + * + * @group datetime_funcs + * @since 1.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a timestamp. */ - def atan2(yName: String, xName: String): Column = - atan2(Column(yName), Column(xName)) + def current_timestamp(): Column = Column.fn("current_timestamp") /** - * @param y - * coordinate on y-axis - * @param xValue - * coordinate on x-axis + * Returns the current timestamp at the start of query evaluation. + * + * @group datetime_funcs + * @since 3.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a timestamp. */ - def atan2(y: Column, xValue: Double): Column = atan2(y, lit(xValue)) + def now(): Column = Column.fn("now") /** - * @param yName - * coordinate on y-axis - * @param xValue - * coordinate on x-axis + * Returns the current timestamp without time zone at the start of query evaluation as a + * timestamp without time zone column. All calls of localtimestamp within the same query return + * the same value. + * + * @group datetime_funcs + * @since 3.3.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a timestamp. */ - def atan2(yName: String, xValue: Double): Column = atan2(Column(yName), xValue) + def localtimestamp(): Column = Column.fn("localtimestamp") /** - * @param yValue - * coordinate on y-axis - * @param x - * coordinate on x-axis + * Converts a date/timestamp/string to a value of string in the format specified by the date + * format given by the second argument. + * + * See Datetime + * Patterns for valid date and time format patterns + * + * @param dateExpr + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp or time. + * @param format + * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to + * a string. * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns + * a column that evaluates to a string. + * @note + * Use specialized functions like [[year]] whenever possible as they benefit from a + * specialized implementation. + * @throws IllegalArgumentException + * if the `format` pattern is invalid + * @group datetime_funcs + * @since 1.5.0 */ - def atan2(yValue: Double, x: Column): Column = atan2(lit(yValue), x) + def date_format(dateExpr: Column, format: String): Column = + Column.fn("date_format", dateExpr, lit(format)) /** - * @param yValue - * coordinate on y-axis - * @param xName - * coordinate on x-axis + * Returns the date that is `days` days after `start` + * + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * The number of days to add to `start`, can be negative to subtract days. A column that + * evaluates to an integer, short, or byte. * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName)) + def date_add(start: Column, days: Int): Column = date_add(start, lit(days)) /** - * @param e - * target column to compute on. A column that evaluates to a numeric. - * @return - * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. + * Returns the date that is `days` days after `start` * - * @group math_funcs - * @since 3.1.0 - */ - def atanh(e: Column): Column = Column.fn("atanh", e) - - /** - * @param columnName - * target column to compute on. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 3.1.0 + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 3.0.0 */ - def atanh(columnName: String): Column = atanh(Column(columnName)) + def date_add(start: Column, days: Column): Column = Column.fn("date_add", start, days) /** - * An expression that returns the string representation of the binary value of the given long - * column. For example, bin("12") returns "1100". + * Returns the date that is `days` days after `start` * - * @param e - * target column to work on. A column that evaluates to an integral. - * @group math_funcs - * @since 1.5.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a string. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 3.5.0 */ - def bin(e: Column): Column = Column.fn("bin", e) + def dateadd(start: Column, days: Column): Column = Column.fn("dateadd", start, days) /** - * An expression that returns the string representation of the binary value of the given long - * column. For example, bin("12") returns "1100". + * Returns the date that is `days` days before `start` * - * @param columnName - * target column to work on. - * @group math_funcs - * @since 1.5.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * The number of days to subtract from `start`, can be negative to add days. A column that + * evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a string. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def bin(columnName: String): Column = bin(Column(columnName)) + def date_sub(start: Column, days: Int): Column = date_sub(start, lit(days)) /** - * Computes the cube-root of the given value. + * Returns the date that is `days` days before `start` * - * @param e - * target column to compute on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * A column of the number of days to subtract from `start`, can be negative to add days. A + * column that evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a double. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 3.0.0 */ - def cbrt(e: Column): Column = Column.fn("cbrt", e) + def date_sub(start: Column, days: Column): Column = + Column.fn("date_sub", start, days) /** - * Computes the cube-root of the given column. + * Returns the number of days from `start` to `end`. * - * @param columnName - * target column to compute on. - * @group math_funcs - * @since 1.4.0 + * Only considers the date part of the input. For example: + * {{{ + * datediff("2018-01-10 00:00:00", "2018-01-09 23:59:59") + * // returns 1 + * }}} + * + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * Returns a column that evaluates to a double. + * An integer, or null if either `end` or `start` were strings that could not be cast to a + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def cbrt(columnName: String): Column = cbrt(Column(columnName)) + def datediff(end: Column, start: Column): Column = Column.fn("datediff", end, start) /** - * Computes the ceiling of the given value of `e` to `scale` decimal places. + * Returns the number of days from `start` to `end`. * - * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @param scale - * parameter to control the rounding behavior. A column that evaluates to an integral. Must be - * a constant. - * @group math_funcs - * @since 3.3.0 + * Only considers the date part of the input. For example: + * {{{ + * date_diff("2018-01-10 00:00:00", "2018-01-09 23:59:59") + * // returns 1 + * }}} + * + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * Returns a column that evaluates to a long or decimal. + * An integer, or null if either `end` or `start` were strings that could not be cast to a + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. + * @group datetime_funcs + * @since 3.5.0 */ - def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) + def date_diff(end: Column, start: Column): Column = Column.fn("date_diff", end, start) /** - * Computes the ceiling of the given value of `e` to 0 decimal places. + * Create date from the number of `days` since 1970-01-01. * - * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param days + * The number of days since 1970-01-01. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column that evaluates to a date. */ - def ceil(e: Column): Column = Column.fn("ceil", e) + def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) /** - * Computes the ceiling of the given value of `columnName` to 0 decimal places. - * - * @param columnName - * the value to compute the ceiling on. - * @group math_funcs - * @since 1.4.0 + * Extracts the year as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the year from. A column that evaluates to a date, + * timestamp or string. * @return - * Returns a column that evaluates to a long or decimal. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def ceil(columnName: String): Column = ceil(Column(columnName)) + def year(e: Column): Column = Column.fn("year", e) /** - * Computes the ceiling of the given value of `e` to `scale` decimal places. - * + * Extracts the quarter as an integer from a given date/timestamp/string. * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @param scale - * parameter to control the rounding behavior. A column that evaluates to an integer. Must be - * a constant. - * @group math_funcs - * @since 3.5.0 + * The date, timestamp or string to extract the quarter from. A column that evaluates to a + * date, timestamp or string. * @return - * Returns a column that evaluates to a long or decimal. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) + def quarter(e: Column): Column = Column.fn("quarter", e) /** - * Computes the ceiling of the given value of `e` to 0 decimal places. - * + * Extracts the month as an integer from a given date/timestamp/string. * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @group math_funcs - * @since 3.5.0 + * The date, timestamp or string to extract the month from. A column that evaluates to a date, + * timestamp or string. * @return - * Returns a column that evaluates to a long or decimal. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def ceiling(e: Column): Column = Column.fn("ceiling", e) + def month(e: Column): Column = Column.fn("month", e) /** - * Convert a number in a string column from one base to another. - * - * @param num - * a column to convert base for. A column that evaluates to a string. - * @param fromBase - * from base number. A column that evaluates to an integer. - * @param toBase - * to base number. A column that evaluates to an integer. - * @group math_funcs - * @since 1.5.0 + * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 + * for a Sunday through to 7 for a Saturday + * @param e + * The date, timestamp or string to extract the day of the week from. A column that evaluates + * to a date, timestamp or string. * @return - * Returns a column that evaluates to a string. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 2.3.0 */ - def conv(num: Column, fromBase: Int, toBase: Int): Column = - Column.fn("conv", num, lit(fromBase), lit(toBase)) + def dayofweek(e: Column): Column = Column.fn("dayofweek", e) /** + * Extracts the day of the month as an integer from a given date/timestamp/string. * @param e - * angle in radians. A column that evaluates to a double. - * @return - * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that - * evaluates to a double. - * - * @group math_funcs - * @since 1.4.0 - */ - def cos(e: Column): Column = Column.fn("cos", e) - - /** - * @param columnName - * angle in radians. A column that evaluates to a double. + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def cos(columnName: String): Column = cos(Column(columnName)) + def dayofmonth(e: Column): Column = Column.fn("dayofmonth", e) /** + * Extracts the day of the month as an integer from a given date/timestamp/string. * @param e - * hyperbolic angle. A column that evaluates to a double. - * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column - * that evaluates to a double. - * - * @group math_funcs - * @since 1.4.0 - */ - def cosh(e: Column): Column = Column.fn("cosh", e) - - /** - * @param columnName - * hyperbolic angle + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column - * that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 3.5.0 */ - def cosh(columnName: String): Column = cosh(Column(columnName)) + def day(e: Column): Column = Column.fn("day", e) /** + * Extracts the day of the year as an integer from a given date/timestamp/string. * @param e - * angle in radians. A column that evaluates to a double. + * The date, timestamp or string to extract the day of the year from. A column that evaluates + * to a date, timestamp or string. * @return - * cotangent of the angle. Returns a column that evaluates to a double. - * - * @group math_funcs - * @since 3.3.0 + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def cot(e: Column): Column = Column.fn("cot", e) + def dayofyear(e: Column): Column = Column.fn("dayofyear", e) /** + * Extracts the hours as an integer from a given date/time/timestamp/string. The input may also + * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in + * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @param e - * angle in radians. A column that evaluates to a double. + * The column to extract the hours from. A column that evaluates to a date, time, timestamp or + * string. * @return - * cosecant of the angle. Returns a column that evaluates to a double. - * - * @group math_funcs - * @since 3.3.0 + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def csc(e: Column): Column = Column.fn("csc", e) + def hour(e: Column): Column = Column.fn("hour", e) /** - * Returns Euler's number. + * Extracts a part of the date/timestamp or interval source. * - * @group math_funcs - * @since 3.5.0 + * @param field + * selects which part of the source should be extracted. + * @param source + * a date, time, timestamp or interval column from where `field` should be extracted. * @return - * Returns a column that evaluates to a double. + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. + * @group datetime_funcs + * @since 3.5.0 */ - def e(): Column = Column.fn("e") + def extract(field: Column, source: Column): Column = { + Column.fn("extract", field, source) + } /** - * Computes the exponential of the given value. + * Extracts a part of the date/timestamp or interval source. * - * @param e - * target column to compute on. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param field + * selects which part of the source should be extracted, and supported string values are as + * same as the fields of the equivalent function `extract`. + * @param source + * a date/timestamp or time or interval column from where `field` should be extracted. * @return - * Returns a column that evaluates to a double. + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. + * @group datetime_funcs + * @since 3.5.0 */ - def exp(e: Column): Column = Column.fn("exp", e) + def date_part(field: Column, source: Column): Column = { + Column.fn("date_part", field, source) + } /** - * Computes the exponential of the given column. + * Extracts a part of the date/timestamp or interval source. * - * @param columnName - * target column to compute on. - * @group math_funcs - * @since 1.4.0 + * @param field + * selects which part of the source should be extracted, and supported string values are as + * same as the fields of the equivalent function `EXTRACT`. + * @param source + * a date/timestamp or interval column from where `field` should be extracted. * @return - * Returns a column that evaluates to a double. + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. + * @group datetime_funcs + * @since 3.5.0 */ - def exp(columnName: String): Column = exp(Column(columnName)) + def datepart(field: Column, source: Column): Column = { + Column.fn("datepart", field, source) + } /** - * Computes the exponential of the given value minus one. + * Returns the last day of the month which the given date belongs to. For example, input + * "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015. * * @param e - * column to calculate exponential for. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 - * @return - * Returns a column that evaluates to a double. - */ - def expm1(e: Column): Column = Column.fn("expm1", e) - - /** - * Computes the exponential of the given column minus one. - * - * @param columnName - * column name to calculate exponential for. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * Returns a column that evaluates to a double. + * A date, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def expm1(columnName: String): Column = expm1(Column(columnName)) + def last_day(e: Column): Column = Column.fn("last_day", e) /** - * Computes the factorial of the given value. - * + * Extracts the minutes as an integer from a given date/time/timestamp/string. The input may + * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in + * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @param e - * a column to calculate factorial for. A column that evaluates to an integral. - * @group math_funcs - * @since 1.5.0 + * The column to extract the minutes from. A column that evaluates to a date, time, timestamp + * or string. * @return - * Returns a column that evaluates to a long. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def factorial(e: Column): Column = Column.fn("factorial", e) + def minute(e: Column): Column = Column.fn("minute", e) /** - * Computes the floor of the given value of `e` to `scale` decimal places. + * Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). * * @param e - * the target column to compute the floor on. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to control the rounding behavior. A column that evaluates to - * an integral. - * @group math_funcs - * @since 3.3.0 + * The column to extract the day of the week from. A column that evaluates to a date, + * timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column that evaluates to an integer. */ - def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) + def weekday(e: Column): Column = Column.fn("weekday", e) /** - * Computes the floor of the given value of `e` to 0 decimal places. - * - * @param e - * the target column to compute the floor on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param year + * The year to build the date. A column that evaluates to an integral. + * @param month + * The month to build the date. A column that evaluates to an integral. + * @param day + * The day to build the date. A column that evaluates to an integral. * @return - * Returns a column that evaluates to a long or decimal. + * A date created from year, month and day fields. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 3.3.0 */ - def floor(e: Column): Column = Column.fn("floor", e) + def make_date(year: Column, month: Column, day: Column): Column = + Column.fn("make_date", year, month, day) /** - * Computes the floor of the given column value to 0 decimal places. + * Returns number of months between dates `start` and `end`. * - * @param columnName - * the target column name to compute the floor on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * A whole number is returned if both inputs have the same day of month or both are the last day + * of their respective months. Otherwise, the difference is calculated assuming 31 days per + * month. + * + * For example: + * {{{ + * months_between("2017-11-14", "2017-07-14") // returns 4.0 + * months_between("2017-01-01", "2017-01-10") // returns 0.29032258 + * months_between("2017-06-01", "2017-06-16 12:00:00") // returns -0.5 + * }}} + * + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can cast to a + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * timestamp. * @return - * Returns a column that evaluates to a long or decimal. + * A double, or null if either `end` or `start` were strings that could not be cast to a + * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a + * double. + * @group datetime_funcs + * @since 1.5.0 */ - def floor(columnName: String): Column = floor(Column(columnName)) + def months_between(end: Column, start: Column): Column = + Column.fn("months_between", end, start) /** - * Returns the greatest value of the list of values, skipping null values. This function takes - * at least 2 parameters. It will return null iff all parameters are null. - * - * @param exprs - * columns to check for greatest value. A column that evaluates to any type. - * @group math_funcs - * @since 1.5.0 + * Returns number of months between dates `end` and `start`. If `roundOff` is set to true, the + * result is rounded off to 8 digits; it is not rounded otherwise. + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can cast to a + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * timestamp. + * @param roundOff + * Whether to round off the result to 8 digits. A column that evaluates to a boolean. Must be + * a constant. + * @group datetime_funcs + * @since 2.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - @scala.annotation.varargs - def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) + def months_between(end: Column, start: Column, roundOff: Boolean): Column = + Column.fn("months_between", end, start, lit(roundOff)) /** - * Returns the greatest value of the list of column names, skipping null values. This function - * takes at least 2 parameters. It will return null iff all parameters are null. + * Returns the first date which is later than the value of the `date` column that is on the + * specified day of the week. * - * @param columnName - * the first column name to check for greatest value. A column of a comparable type. - * @param columnNames - * the remaining column names to check for greatest value. Columns of a comparable type. - * @group math_funcs - * @since 1.5.0 + * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first + * Sunday after 2015-07-27. + * + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param dayOfWeek + * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column + * that evaluates to a string. * @return - * Returns a column of the same type as the input. + * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` + * was an invalid value. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - @scala.annotation.varargs - def greatest(columnName: String, columnNames: String*): Column = { - greatest((columnName +: columnNames).map(Column.apply): _*) - } + def next_day(date: Column, dayOfWeek: String): Column = next_day(date, lit(dayOfWeek)) /** - * Computes hex value of the given column. + * Returns the first date which is later than the value of the `date` column that is on the + * specified day of the week. * - * @param column - * target column to work on. A column that evaluates to an integral, string or binary. - * @group math_funcs - * @since 1.5.0 + * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first + * Sunday after 2015-07-27. + * + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param dayOfWeek + * A column of the day of week. Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", + * "Fri", "Sat", "Sun". A column that evaluates to a string. * @return - * Returns a column that evaluates to a string. + * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` + * was an invalid value. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 3.2.0 */ - def hex(column: Column): Column = Column.fn("hex", column) + def next_day(date: Column, dayOfWeek: Column): Column = + Column.fn("next_day", date, dayOfWeek) /** - * Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to - * the byte representation of number. - * - * @param column - * target column to work on. A column that evaluates to a string. - * @group math_funcs - * @since 1.5.0 + * Extracts the seconds as an integer from a given date/time/timestamp/string. The input may + * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in + * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * @param e + * The column to extract the seconds from. A column that evaluates to a date, time, timestamp + * or string. * @return - * Returns a column that evaluates to a binary. + * An integer, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def unhex(column: Column): Column = Column.fn("unhex", column) + def second(e: Column): Column = Column.fn("second", e) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Extracts the week number as an integer from a given date/timestamp/string. * - * @param l - * a leg. A column that evaluates to a numeric. - * @param r - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * A week is considered to start on a Monday and week 1 is the first week with more than 3 days, + * as defined by ISO 8601 + * + * @param e + * The column to extract the week number from. A column that evaluates to a date, timestamp or + * string. * @return - * Returns a column that evaluates to a double. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) + def weekofyear(e: Column): Column = Column.fn("weekofyear", e) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string + * representing the timestamp of that moment in the current system time zone in the yyyy-MM-dd + * HH:mm:ss format. * - * @param l - * a leg. A column that evaluates to a numeric. - * @param rightName - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param ut + * A number of a type that is castable to a long, such as string or integer. Can be negative + * for timestamps before the unix epoch * @return - * Returns a column that evaluates to a double. + * A string, or null if the input was a string that could not be cast to a long. Returns a + * column that evaluates to a string. + * @group datetime_funcs + * @since 1.5.0 */ - def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) + def from_unixtime(ut: Column): Column = Column.fn("from_unixtime", ut) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string + * representing the timestamp of that moment in the current system time zone in the given + * format. * - * @param leftName - * a leg. A column that evaluates to a numeric. - * @param r - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * See Datetime + * Patterns for valid date and time format patterns + * + * @param ut + * A number of a type that is castable to a long, such as string or integer. Can be negative + * for timestamps before the unix epoch + * @param f + * A date time pattern that the input will be formatted to * @return - * Returns a column that evaluates to a double. + * A string, or null if `ut` was a string that could not be cast to a long or `f` was an + * invalid date time pattern. Returns a column that evaluates to a string. + * @group datetime_funcs + * @since 1.5.0 */ - def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) + def from_unixtime(ut: Column, f: String): Column = + Column.fn("from_unixtime", ut, lit(f)) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Returns the current Unix timestamp (in seconds) as a long. * - * @param leftName - * a leg. A column that evaluates to a numeric. - * @param rightName - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @note + * All calls of `unix_timestamp` within the same query return the same value (i.e. the current + * timestamp is calculated at the start of query evaluation). + * + * @group datetime_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long. */ - def hypot(leftName: String, rightName: String): Column = - hypot(Column(leftName), Column(rightName)) + def unix_timestamp(): Column = unix_timestamp(current_timestamp()) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds), using the + * default timezone and the default locale. * - * @param l - * a leg. A column that evaluates to a numeric. - * @param r - * b leg. A column that evaluates to a numeric. Must be a constant. - * @group math_funcs - * @since 1.4.0 + * @param s + * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` + * format * @return - * Returns a column that evaluates to a double. + * A long, or null if the input was a string not of the correct format. Returns a column that + * evaluates to a long. + * @group datetime_funcs + * @since 1.5.0 */ - def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) + def unix_timestamp(s: Column): Column = Column.fn("unix_timestamp", s) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Converts time string with given pattern to Unix timestamp (in seconds). * - * @param leftName - * The a leg of the triangle. A column that evaluates to a numeric. - * @param r - * The b leg of the triangle. A column that evaluates to a numeric. Must be a constant. - * @group math_funcs - * @since 1.4.0 + * See Datetime + * Patterns for valid date and time format patterns + * + * @param s + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * string, date, or timestamp. + * @param p + * A date time pattern detailing the format of `s` when `s` is a string. A column that + * evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid + * format. Returns a column that evaluates to a long. + * @group datetime_funcs + * @since 1.5.0 */ - def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) + def unix_timestamp(s: Column, p: String): Column = + Column.fn("unix_timestamp", s, lit(p)) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Parses a string value to a time value. * - * @param l - * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. - * @param r - * The b leg of the triangle. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param str + * A string to be parsed to time. A column that evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. + * + * @group datetime_funcs + * @since 4.1.0 */ - def hypot(l: Double, r: Column): Column = hypot(lit(l), r) + def to_time(str: Column): Column = { + Column.fn("to_time", str) + } /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Parses a string value to a time value. * - * @param l - * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. - * @param rightName - * The b leg of the triangle. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 - * @return - * Returns a column that evaluates to a double. - */ - def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) - - /** - * Returns the least value of the list of values, skipping null values. This function takes at - * least 2 parameters. It will return null iff all parameters are null. + * See Datetime + * Patterns for valid time format patterns. * - * @param exprs - * The values to be compared. Columns that evaluate to a comparable type. - * @group math_funcs - * @since 1.5.0 + * @param str + * A string to be parsed to time. + * @param format + * A time format pattern to follow. A column that evaluates to a string. * @return - * Returns a column of the same type as the input. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. + * @group datetime_funcs + * @since 4.1.0 */ - @scala.annotation.varargs - def least(exprs: Column*): Column = Column.fn("least", exprs: _*) + def to_time(str: Column, format: Column): Column = { + Column.fn("to_time", str, format) + } /** - * Returns the least value of the list of column names, skipping null values. This function - * takes at least 2 parameters. It will return null iff all parameters are null. + * Converts to a timestamp by casting rules to `TimestampType`. * - * @param columnName - * The name of the first column to be compared. A column of a comparable type. - * @param columnNames - * The names of the remaining columns to be compared. Columns of a comparable type. - * @group math_funcs - * @since 1.5.0 + * @param s + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. * @return - * Returns a column of the same type as the input. + * A timestamp, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 2.2.0 */ - @scala.annotation.varargs - def least(columnName: String, columnNames: String*): Column = { - least((columnName +: columnNames).map(Column.apply): _*) - } + def to_timestamp(s: Column): Column = Column.fn("to_timestamp", s) /** - * Computes the natural logarithm of the given value. + * Converts time string with the given pattern to timestamp. * - * @param e - * The value to compute the natural logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 3.5.0 + * See Datetime + * Patterns for valid date and time format patterns + * + * @param s + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. + * @param fmt + * A date time pattern detailing the format of `s` when `s` is a string. A column that + * evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was + * an invalid format. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 2.2.0 */ - def ln(e: Column): Column = Column.fn("ln", e) + def to_timestamp(s: Column, fmt: String): Column = Column.fn("to_timestamp", s, lit(fmt)) /** - * Computes the natural logarithm of the given value. + * Parses a string value to a time value. * - * @param e - * The value to compute the natural logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param str + * A string to be parsed to time. A column that evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. + * + * @group datetime_funcs + * @since 4.1.0 */ - def log(e: Column): Column = ln(e) + def try_to_time(str: Column): Column = { + Column.fn("try_to_time", str) + } /** - * Computes the natural logarithm of the given column. + * Parses a string value to a time value. * - * @param columnName - * The name of the column to compute the natural logarithm of. A column that evaluates to a - * numeric. - * @group math_funcs - * @since 1.4.0 + * See Datetime + * Patterns for valid time format patterns. + * + * @param str + * A string to be parsed to time. + * @param format + * A time format pattern to follow. A column that evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 */ - def log(columnName: String): Column = log(Column(columnName)) + def try_to_time(str: Column, format: Column): Column = { + Column.fn("try_to_time", str, format) + } /** - * Returns the first argument-base logarithm of the second argument. + * Parses the `s` with the `format` to a timestamp. The function always returns null on an + * invalid input with`/`without ANSI SQL mode enabled. The result data type is consistent with + * the value of configuration `spark.sql.timestampType`. * - * @param base - * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. - * @param a - * The value to compute the logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. + * @param format + * Format to use to convert timestamp values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a timestamp. */ - def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) + def try_to_timestamp(s: Column, format: Column): Column = + Column.fn("try_to_timestamp", s, format) /** - * Returns the first argument-base logarithm of the second argument. + * Parses the `s` to a timestamp. The function always returns null on an invalid input + * with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The result + * data type is consistent with the value of configuration `spark.sql.timestampType`. * - * @param base - * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. - * @param columnName - * The name of the column to compute the logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a timestamp. */ - def log(base: Double, columnName: String): Column = log(base, Column(columnName)) + def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) /** - * Computes the logarithm of the given value in base 10. + * Converts the column into `DateType` by casting rules to `DateType`. * * @param e - * The value to compute the base-10 logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @group datetime_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a date. */ - def log10(e: Column): Column = Column.fn("log10", e) + def to_date(e: Column): Column = Column.fn("to_date", e) /** - * Computes the logarithm of the given value in base 10. + * Converts the column into a `DateType` with a specified format * - * @param columnName - * The name of the column to compute the base-10 logarithm of. A column that evaluates to a - * numeric. - * @group math_funcs - * @since 1.4.0 + * See Datetime + * Patterns for valid date and time format patterns + * + * @param e + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * string, date, or timestamp. + * @param fmt + * A date time pattern detailing the format of `e` when `e`is a string. A column that + * evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an + * invalid format. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 2.2.0 */ - def log10(columnName: String): Column = log10(Column(columnName)) + def to_date(e: Column, fmt: String): Column = Column.fn("to_date", e, lit(fmt)) /** - * Computes the natural logarithm of the given value plus one. + * This is a special version of `to_date` that performs the same operation, but returns a NULL + * value instead of raising an error if date cannot be created. * * @param e - * The value to compute the natural logarithm of the value plus one. A column that evaluates - * to a numeric. - * @group math_funcs - * @since 1.4.0 + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a date. */ - def log1p(e: Column): Column = Column.fn("log1p", e) + def try_to_date(e: Column): Column = Column.fn("try_to_date", e) /** - * Computes the natural logarithm of the given column plus one. + * This is a special version of `to_date` that performs the same operation, but returns a NULL + * value instead of raising an error if date cannot be created. * - * @param columnName - * The name of the column to compute the natural logarithm of the value plus one. A column - * that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param e + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @param fmt + * Format to use to convert date values. A column that evaluates to a string. Must be a + * constant. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a date. */ - def log1p(columnName: String): Column = log1p(Column(columnName)) + def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) /** - * Computes the logarithm of the given column in base 2. + * Returns the number of days since 1970-01-01. * - * @param expr - * The value to compute the base-2 logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.5.0 + * @param e + * Input column of values to convert. A column that evaluates to a date. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def log2(expr: Column): Column = Column.fn("log2", expr) + def unix_date(e: Column): Column = Column.fn("unix_date", e) /** - * Computes the logarithm of the given value in base 2. + * Returns the number of microseconds since 1970-01-01 00:00:00 UTC. * - * @param columnName - * a column to calculate logarithm for. A column that evaluates to a double. - * @group math_funcs - * @since 1.5.0 + * @param e + * Input column of values to convert. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long. */ - def log2(columnName: String): Column = log2(Column(columnName)) + def unix_micros(e: Column): Column = Column.fn("unix_micros", e) /** - * Returns the negated value. + * Returns the number of nanoseconds since 1970-01-01 00:00:00 UTC for a nanosecond-precision + * timestamp (`TIMESTAMP_LTZ(p)` / `TIMESTAMP_NTZ(p)`, `p` in `[7, 9]`). The result is a + * lossless `DECIMAL(21, 0)`. * * @param e - * column to calculate negative value for. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 3.5.0 + * input column of nanosecond-precision timestamp values to convert. A column that evaluates + * to a timestamp. + * @group datetime_funcs + * @since 4.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a decimal. */ - def negative(e: Column): Column = Column.fn("negative", e) + def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) /** - * Returns Pi. + * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of + * precision. * - * @group math_funcs + * @param e + * input column of values to convert. A column that evaluates to a timestamp. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long. */ - def pi(): Column = Column.fn("pi") + def unix_millis(e: Column): Column = Column.fn("unix_millis", e) /** - * Returns the value. + * Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of + * precision. * * @param e - * input value column. A column that evaluates to a numeric or interval. - * @group math_funcs + * input column of values to convert. A column that evaluates to a timestamp. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def positive(e: Column): Column = Column.fn("positive", e) + def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) /** - * Returns the value of the first argument raised to the power of the second argument. + * Returns date truncated to the unit specified by the format. * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 - * @return - * Returns a column that evaluates to a double. - */ - def pow(l: Column, r: Column): Column = Column.fn("power", l, r) - - /** - * Returns the value of the first argument raised to the power of the second argument. + * For example, `trunc("2018-11-19 12:01:19", "year")` returns 2018-01-01 * - * @param l - * the base number. A column that evaluates to a double. - * @param rightName - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 - * @return - * Returns a column that evaluates to a double. - */ - def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) - - /** - * Returns the value of the first argument raised to the power of the second argument. + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param format: + * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month + * Other options are: 'week', 'quarter'. A column that evaluates to a string. * - * @param leftName - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 * @return - * Returns a column that evaluates to a double. + * A date, or null if `date` was a string that could not be cast to a date or `format` was an + * invalid value. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) + def trunc(date: Column, format: String): Column = Column.fn("trunc", date, lit(format)) /** - * Returns the value of the first argument raised to the power of the second argument. + * Returns timestamp truncated to the unit specified by the format. * - * @param leftName - * the base number. - * @param rightName - * the exponent number. - * @group math_funcs - * @since 1.4.0 - * @return - * Returns a column that evaluates to a double. - */ - def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) - - /** - * Returns the value of the first argument raised to the power of the second argument. + * For example, `date_trunc("year", "2018-11-19 12:01:19")` returns 2018-01-01 00:00:00 * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param format: + * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', + * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', + * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. + * @param timestamp + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @return - * Returns a column that evaluates to a double. + * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or + * `format` was an invalid value. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 2.3.0 */ - def pow(l: Column, r: Double): Column = pow(l, lit(r)) + def date_trunc(format: String, timestamp: Column): Column = + Column.fn("date_trunc", lit(format), timestamp) /** - * Returns the value of the first argument raised to the power of the second argument. + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders + * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 + * 03:40:00.0'. * - * @param leftName - * the base number. - * @param r - * the exponent number. - * @group math_funcs - * @since 1.4.0 + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. It should be in + * the format of either region-based zone IDs or zone offsets. Region IDs must have the form + * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was + * an invalid value. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 1.5.0 */ - def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) + def from_utc_timestamp(ts: Column, tz: String): Column = from_utc_timestamp(ts, lit(tz)) /** - * Returns the value of the first argument raised to the power of the second argument. - * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders + * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 + * 03:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. + * @group datetime_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a timestamp. */ - def pow(l: Double, r: Column): Column = pow(lit(l), r) + def from_utc_timestamp(ts: Column, tz: Column): Column = + Column.fn("from_utc_timestamp", ts, tz) /** - * Returns the value of the first argument raised to the power of the second argument. + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time + * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield + * '2017-07-14 01:40:00.0'. * - * @param l - * the base number. - * @param rightName - * the exponent number. - * @group math_funcs - * @since 1.4.0 + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. It should be in + * the format of either region-based zone IDs or zone offsets. Region IDs must have the form + * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return - * Returns a column that evaluates to a double. + * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was + * an invalid value. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 1.5.0 */ - def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) + def to_utc_timestamp(ts: Column, tz: String): Column = to_utc_timestamp(ts, lit(tz)) /** - * Returns the value of the first argument raised to the power of the second argument. - * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 3.5.0 + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time + * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield + * '2017-07-14 01:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. + * @group datetime_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a timestamp. */ - def power(l: Column, r: Column): Column = Column.fn("power", l, r) + def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) /** - * Returns the positive value of dividend mod divisor. + * Bucketize rows into one or more time windows given a timestamp specifying column. Window + * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window + * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in + * the order of months are not supported. The following example takes the average stock price + * for a one minute window every 10 seconds starting 5 seconds after the hour: * - * @param dividend - * the column that contains dividend, or the specified dividend value. A column that evaluates - * to a numeric. - * @param divisor - * the column that contains divisor, or the specified divisor value. A column that evaluates - * to a numeric. - * @group math_funcs - * @since 1.5.0 - * @return - * Returns a column of the same type as the input. - */ - def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) - - /** - * Returns the double value that is closest in value to the argument and is equal to a - * mathematical integer. + * {{{ + * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType + * df.groupBy(window($"timestamp", "1 minute", "10 seconds", "5 seconds"), $"stockId") + * .agg(mean("price")) + * }}} * - * @param e - * target column to compute on. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * The windows will look like: + * + * {{{ + * 09:00:05-09:01:05 + * 09:00:15-09:01:15 + * 09:00:25-09:01:25 ... + * }}} + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param windowDuration + * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that + * the duration is a fixed length of time, and does not vary over time according to a + * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A + * column that evaluates to a string. + * @param slideDuration + * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will + * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. + * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This + * duration is likewise absolute, and does not vary according to a calendar. A column that + * evaluates to a string. + * @param startTime + * The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals. + * For example, in order to have hourly tumbling windows that start 15 minutes past the hour, + * e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`. A column that + * evaluates to a string. + * + * @group datetime_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a struct. */ - def rint(e: Column): Column = Column.fn("rint", e) + def window( + timeColumn: Column, + windowDuration: String, + slideDuration: String, + startTime: String): Column = + Column.fn("window", timeColumn, lit(windowDuration), lit(slideDuration), lit(startTime)) /** - * Returns the double value that is closest in value to the argument and is equal to a - * mathematical integer. + * Bucketize rows into one or more time windows given a timestamp specifying column. Window + * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window + * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in + * the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 + * UTC. The following example takes the average stock price for a one minute window every 10 + * seconds: * - * @param columnName - * the numeric column name to round to the closest integer. - * @group math_funcs - * @since 1.4.0 + * {{{ + * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType + * df.groupBy(window($"timestamp", "1 minute", "10 seconds"), $"stockId") + * .agg(mean("price")) + * }}} + * + * The windows will look like: + * + * {{{ + * 09:00:00-09:01:00 + * 09:00:10-09:01:10 + * 09:00:20-09:01:20 ... + * }}} + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param windowDuration + * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that + * the duration is a fixed length of time, and does not vary over time according to a + * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A + * column that evaluates to a string. + * @param slideDuration + * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will + * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. + * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This + * duration is likewise absolute, and does not vary according to a calendar. A column that + * evaluates to a string. + * + * @group datetime_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a struct. */ - def rint(columnName: String): Column = rint(Column(columnName)) + def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { + window(timeColumn, windowDuration, slideDuration, "0 second") + } /** - * Returns the value of the column `e` rounded to 0 decimal places with HALF_UP round mode. + * Generates tumbling time windows given a timestamp specifying column. Window starts are + * inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) + * but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of + * months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC. The + * following example takes the average stock price for a one minute tumbling window: * - * @param e - * the value to round. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.5.0 + * {{{ + * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType + * df.groupBy(window($"timestamp", "1 minute"), $"stockId") + * .agg(mean("price")) + * }}} + * + * The windows will look like: + * + * {{{ + * 09:00:00-09:01:00 + * 09:01:00-09:02:00 + * 09:02:00-09:03:00 ... + * }}} + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param windowDuration + * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column + * that evaluates to a string. + * + * @group datetime_funcs + * @since 2.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a struct. */ - def round(e: Column): Column = round(e, 0) + def window(timeColumn: Column, windowDuration: String): Column = { + window(timeColumn, windowDuration, windowDuration, "0 second") + } /** - * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Extracts the event time from the window column. * - * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs - * @since 1.5.0 + * The window column is of StructType { start: Timestamp, end: Timestamp } where start is + * inclusive and end is exclusive. Since event time can support microsecond precision, + * window_time(window) = window.end - 1 microsecond. + * + * @param windowColumn + * The window column (typically produced by window aggregation) of type StructType { start: + * Timestamp, end: Timestamp }. A column that evaluates to a struct. + * + * @group datetime_funcs + * @since 3.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a timestamp. */ - def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) + def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) /** - * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Generates session window given a timestamp specifying column. * - * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs - * @since 4.0.0 + * Session window is one of dynamic windows, which means the length of window is varying + * according to the given inputs. The length of session window is defined as "the timestamp of + * latest input of the session + gap duration", so when the new inputs are bound to the current + * session window, the end time of session window can be expanded according to the new inputs. + * + * Windows can support microsecond precision. gapDuration in the order of months are not + * supported. + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param gapDuration + * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column + * that evaluates to a string. + * + * @group datetime_funcs + * @since 3.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a struct. */ - def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) + def session_window(timeColumn: Column, gapDuration: String): Column = + session_window(timeColumn, lit(gapDuration)) /** - * Truncates the value of `e` toward zero to 0 decimal places. + * Generates session window given a timestamp specifying column. * - * @param e - * the value to truncate. A column that evaluates to a numeric. + * Session window is one of dynamic windows, which means the length of window is varying + * according to the given inputs. For static gap duration, the length of session window is + * defined as "the timestamp of latest input of the session + gap duration", so when the new + * inputs are bound to the current session window, the end time of session window can be + * expanded according to the new inputs. + * + * Besides a static gap duration value, users can also provide an expression to specify gap + * duration dynamically based on the input row. With dynamic gap duration, the closing of a + * session window does not depend on the latest input anymore. A session window's range is the + * union of all events' ranges which are determined by event start time and evaluated gap + * duration during the query execution. Note that the rows with negative or zero gap duration + * will be filtered out from the aggregation. + * + * Windows can support microsecond precision. gapDuration in the order of months are not + * supported. + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param gapDuration + * A column specifying the timeout of the session. It could be static value, e.g. `10 + * minutes`, `1 second`, or an expression/UDF that specifies gap duration dynamically based on + * the input row. A column that evaluates to a string or interval. + * + * @group datetime_funcs + * @since 3.2.0 * @return - * Returns a column of the same type as the input, except that a decimal input may return a - * decimal of different precision and scale. - * @group math_funcs - * @since 4.4.0 + * Returns a column that evaluates to a struct. */ - def truncate(e: Column): Column = truncate(e, 0) + def session_window(timeColumn: Column, gapDuration: Column): Column = + Column.fn("session_window", timeColumn, gapDuration) /** - * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than - * or equal to 0, or to the left of the decimal point when `scale` is less than 0. - * + * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. * @param e - * the value to truncate. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to keep. A column that evaluates to an integral. Must be a - * constant. + * unix time values. A column that evaluates to a numeric. + * @group datetime_funcs + * @since 3.1.0 * @return - * Returns a column of the same type as the input, except that a decimal input may return a - * decimal of different precision and scale. - * @group math_funcs - * @since 4.4.0 + * Returns a column that evaluates to a timestamp. */ - def truncate(e: Column, scale: Int): Column = Column.fn("truncate", e, lit(scale)) + def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) /** - * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than - * or equal to 0, or to the left of the decimal point when `scale` is less than 0. + * Creates timestamp from the number of milliseconds since UTC epoch. * * @param e - * the value to truncate. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to keep. A column that evaluates to an integral. Must be a - * constant. + * unix time values. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input, except that a decimal input may return a - * decimal of different precision and scale. - * @group math_funcs - * @since 4.4.0 + * Returns a column that evaluates to a timestamp. */ - def truncate(e: Column, scale: Column): Column = Column.fn("truncate", e, scale) + def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) /** - * Returns the value of the column `e` rounded to 0 decimal places with HALF_EVEN round mode. + * Creates timestamp from the number of microseconds since UTC epoch. * * @param e - * the value to round. A column that evaluates to a numeric. - * @group math_funcs - * @since 2.0.0 + * unix time values. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a timestamp. */ - def bround(e: Column): Column = bround(e, 0) + def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) /** - * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Creates a timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from + * the number of nanoseconds since UTC epoch. * * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs - * @since 2.0.0 + * nanosecond values since the UTC epoch. A column that evaluates to an integral or decimal. + * @group datetime_funcs + * @since 4.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a timestamp. */ - def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) + def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) /** - * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Gets the difference between the timestamps in the specified units by truncating the fraction + * part. * - * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs + * @param unit + * the units of the difference between the given timestamps, e.g. 'YEAR', 'MONTH', 'DAY', + * 'HOUR'. A column that evaluates to a string. Must be a constant. + * @param start + * A timestamp which the expression subtracts from `end`. A column that evaluates to a + * timestamp. + * @param end + * A timestamp from which the expression subtracts `start`. A column that evaluates to a + * timestamp. + * @group datetime_funcs * @since 4.0.0 * @return - * Returns a column of the same type as the input. - */ - def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) - - /** - * @param e - * angle in radians. A column that evaluates to a double. - * @return - * secant of the angle. Returns a column that evaluates to a double. - * - * @group math_funcs - * @since 3.3.0 + * Returns a column that evaluates to a long. */ - def sec(e: Column): Column = Column.fn("sec", e) + def timestamp_diff(unit: String, start: Column, end: Column): Column = + Column.internalFn("timestampdiff", lit(unit), start, end) /** - * Shift the given value numBits left. If the given value is a long value, this function will - * return a long value else it will return an integer value. + * Adds the specified number of units to the given timestamp. * - * @group bitwise_funcs - * @since 1.5.0 + * @param unit + * the units of datetime to add, e.g. 'YEAR', 'MONTH', 'DAY', 'HOUR'. A column that evaluates + * to a string. Must be a constant. + * @param quantity + * the number of units of time to add. A column that evaluates to an integral. + * @param ts + * A timestamp to which to add. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 4.0.0 * @return * Returns a column of the same type as the input. */ - @deprecated("Use shiftleft", "3.2.0") - def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) + def timestamp_add(unit: String, quantity: Column, ts: Column): Column = + Column.internalFn("timestampadd", lit(unit), quantity, ts) /** - * Shift the given value numBits left. If the given value is a long value, this function will - * return a long value else it will return an integer value. + * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets + * aligned to the default origin (1970-01-01 00:00:00). For `TIMESTAMP_NTZ`, bucketing is + * performed in UTC. For `TIMESTAMP`, year-month interval buckets and calendar-day components of + * day-time interval buckets align to the session time zone. * - * @param e - * the value to shift. A column that evaluates to an integral. - * @param numBits - * the number of bits to shift left. A column that evaluates to an integral. Must be a - * constant. - * @group bitwise_funcs - * @since 3.2.0 + * @param bucketSize + * A day-time or year-month interval defining the bucket size. Must be positive and foldable. + * @param ts + * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. + * @group datetime_funcs + * @since 4.2.0 * @return * Returns a column of the same type as the input. */ - def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) + def time_bucket(bucketSize: Column, ts: Column): Column = + Column.fn("time_bucket", bucketSize, ts) /** - * (Signed) shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. + * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets + * aligned to `origin`. For `TIMESTAMP_NTZ`, bucketing is performed in UTC. For `TIMESTAMP`, + * year-month interval buckets and calendar-day components of day-time interval buckets align to + * the session time zone. * - * @group bitwise_funcs - * @since 1.5.0 + * @param bucketSize + * A day-time or year-month interval defining the bucket size. Must be positive and foldable. + * @param ts + * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. + * @param origin + * Alignment anchor. Must be the same type as `ts` and must be foldable. + * @group datetime_funcs + * @since 4.2.0 * @return * Returns a column of the same type as the input. */ - @deprecated("Use shiftright", "3.2.0") - def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) + def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = + Column.fn("time_bucket", bucketSize, ts, origin) /** - * (Signed) shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. + * Returns the difference between two times, measured in specified units. Throws a + * SparkIllegalArgumentException, in case the specified unit is not supported. * - * @param e - * the value to shift. A column that evaluates to an integral. - * @param numBits - * the number of bits to shift right. A column that evaluates to an integral. Must be a - * constant. - * @group bitwise_funcs - * @since 3.2.0 + * @param unit + * A STRING representing the unit of the time difference. Supported units are: "HOUR", + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. + * @param start + * A starting TIME. A column that evaluates to a time. + * @param end + * An ending TIME. A column that evaluates to a time. * @return - * Returns a column of the same type as the input. + * The difference between `end` and `start` times, measured in specified units. Returns a + * column that evaluates to a long. + * @note + * If any of the inputs is `NULL`, the result is `NULL`. + * @group datetime_funcs + * @since 4.1.0 */ - def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) + def time_diff(unit: Column, start: Column, end: Column): Column = { + Column.fn("time_diff", unit, start, end) + } /** - * Unsigned shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. + * Returns `time` truncated to the `unit`. * - * @group bitwise_funcs - * @since 1.5.0 + * @param unit + * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. + * @param time + * A TIME to truncate. A column that evaluates to a time. * @return - * Returns a column of the same type as the input. + * A TIME truncated to the specified unit. Returns a column that evaluates to a time. + * @note + * If any of the inputs is `NULL`, the result is `NULL`. + * @throws IllegalArgumentException + * If the `unit` is not supported. + * @group datetime_funcs + * @since 4.1.0 */ - @deprecated("Use shiftrightunsigned", "3.2.0") - def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) + def time_trunc(unit: Column, time: Column): Column = { + Column.fn("time_trunc", unit, time) + } /** - * Unsigned shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. + * Creates a TIME from the number of seconds since midnight. * * @param e - * the value to shift. A column that evaluates to an integral. - * @param numBits - * the number of bits to shift right. A column that evaluates to an integral. Must be a - * constant. - * @group bitwise_funcs - * @since 3.2.0 + * seconds since midnight (0 to 86399.999999). A column that evaluates to a numeric. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a time. */ - def shiftrightunsigned(e: Column, numBits: Int): Column = - Column.fn("shiftrightunsigned", e, lit(numBits)) + def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) /** - * Computes the signum of the given value. + * Creates a TIME from the number of milliseconds since midnight. * * @param e - * the value to compute the signum of. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 3.5.0 + * milliseconds since midnight (0 to 86399999). A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a time. */ - def sign(e: Column): Column = Column.fn("sign", e) + def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) /** - * Computes the signum of the given value. + * Creates a TIME from the number of microseconds since midnight. * * @param e - * the value to compute the signum of. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 1.4.0 + * microseconds since midnight (0 to 86399999999). A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a time. */ - def signum(e: Column): Column = Column.fn("signum", e) + def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) /** - * Computes the signum of the given column. + * Extracts the number of seconds (including fractional seconds) from a TIME value. Returns a + * DECIMAL(14,6) to preserve microsecond precision. * - * @param columnName - * column to compute the signum on. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 1.4.0 + * @param e + * TIME value to convert. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a decimal. */ - def signum(columnName: String): Column = signum(Column(columnName)) + def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) /** + * Extracts the number of milliseconds since midnight from a TIME value. + * * @param e - * angle in radians. A column that evaluates to a double. + * the TIME value to convert. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.2.0 * @return - * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates - * to a double. - * - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a long. */ - def sin(e: Column): Column = Column.fn("sin", e) + def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) /** - * @param columnName - * angle in radians. A column that evaluates to a double. + * Extracts the number of microseconds since midnight from a TIME value. + * + * @param e + * the TIME value to convert. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.2.0 * @return - * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates - * to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a long. */ - def sin(columnName: String): Column = sin(Column(columnName)) + def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) /** - * @param e - * hyperbolic angle. A column that evaluates to a double. + * Parses the `timestamp` expression with the `format` expression to a timestamp with local time + * zone. Returns null with invalid input. + * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a timestamp. */ - def sinh(e: Column): Column = Column.fn("sinh", e) + def to_timestamp_ltz(timestamp: Column, format: Column): Column = + Column.fn("to_timestamp_ltz", timestamp, format) /** - * @param columnName - * hyperbolic angle. A column that evaluates to a double. + * Parses the `timestamp` expression with the default format to a timestamp with local time + * zone. The default format follows casting rules to a timestamp. Returns null with invalid + * input. + * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a timestamp. */ - def sinh(columnName: String): Column = sinh(Column(columnName)) + def to_timestamp_ltz(timestamp: Column): Column = + Column.fn("to_timestamp_ltz", timestamp) /** - * @param e - * angle in radians. A column that evaluates to a double. - * @return - * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that - * evaluates to a double. + * Parses the `timestamp_str` expression with the `format` expression to a timestamp without + * time zone. Returns null with invalid input. * - * @group math_funcs - * @since 1.4.0 + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. */ - def tan(e: Column): Column = Column.fn("tan", e) + def to_timestamp_ntz(timestamp: Column, format: Column): Column = + Column.fn("to_timestamp_ntz", timestamp, format) /** - * @param columnName - * angle in radians. A column that evaluates to a double. + * Parses the `timestamp` expression with the default format to a timestamp without time zone. + * The default format follows casting rules to a timestamp. Returns null with invalid input. + * + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a timestamp. */ - def tan(columnName: String): Column = tan(Column(columnName)) + def to_timestamp_ntz(timestamp: Column): Column = + Column.fn("to_timestamp_ntz", timestamp) /** - * @param e - * hyperbolic angle. A column that evaluates to a double. + * Returns the UNIX timestamp of the given time. + * + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to convert the time values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a long. */ - def tanh(e: Column): Column = Column.fn("tanh", e) + def to_unix_timestamp(timeExp: Column, format: Column): Column = + Column.fn("to_unix_timestamp", timeExp, format) /** - * @param columnName - * hyperbolic angle. A column that evaluates to a double. + * Returns the UNIX timestamp of the given time. + * + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a long. */ - def tanh(columnName: String): Column = tanh(Column(columnName)) + def to_unix_timestamp(timeExp: Column): Column = + Column.fn("to_unix_timestamp", timeExp) /** - * @group math_funcs - * @since 1.4.0 + * Extracts the three-letter abbreviated month name from a given date/timestamp/string. + * + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or + * string. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - @deprecated("Use degrees", "2.1.0") - def toDegrees(e: Column): Column = degrees(e) + def monthname(timeExp: Column): Column = + Column.fn("monthname", timeExp) /** - * @group math_funcs - * @since 1.4.0 + * Extracts the three-letter abbreviated day name from a given date/timestamp/string. + * + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or + * string. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - @deprecated("Use degrees", "2.1.0") - def toDegrees(columnName: String): Column = degrees(Column(columnName)) + def dayname(timeExp: Column): Column = + Column.fn("dayname", timeExp) /** - * Converts an angle measured in radians to an approximately equivalent angle measured in - * degrees. + * Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to + * `targetTz`. * - * @param e - * angle in radians. A column that evaluates to a double. + * @param sourceTz + * the time zone for the input timestamp. If it is missed, the current session time zone is + * used as the source time zone. A column that evaluates to a string. + * @param targetTz + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. + * @param sourceTs + * a timestamp without time zone. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that - * evaluates to a double. - * - * @group math_funcs - * @since 2.1.0 + * Returns a column that evaluates to a timestamp. */ - def degrees(e: Column): Column = Column.fn("degrees", e) + def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = + Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) /** - * Converts an angle measured in radians to an approximately equivalent angle measured in - * degrees. + * Converts the timestamp without time zone `sourceTs` from the current time zone to `targetTz`. * - * @param columnName - * angle in radians. A column that evaluates to a double. + * @param targetTz + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. + * @param sourceTs + * a timestamp without time zone. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 2.1.0 + * Returns a column that evaluates to a timestamp. */ - def degrees(columnName: String): Column = degrees(Column(columnName)) + def convert_timezone(targetTz: Column, sourceTs: Column): Column = + Column.fn("convert_timezone", targetTz, sourceTs) /** - * @group math_funcs - * @since 1.4.0 + * Make DayTimeIntervalType duration from days, hours, mins and secs. + * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an interval. */ - @deprecated("Use radians", "2.1.0") - def toRadians(e: Column): Column = radians(e) + def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = + Column.fn("make_dt_interval", days, hours, mins, secs) /** - * @group math_funcs - * @since 1.4.0 + * Make DayTimeIntervalType duration from days, hours and mins. + * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an interval. */ - @deprecated("Use radians", "2.1.0") - def toRadians(columnName: String): Column = radians(Column(columnName)) + def make_dt_interval(days: Column, hours: Column, mins: Column): Column = + Column.fn("make_dt_interval", days, hours, mins) /** - * Converts an angle measured in degrees to an approximately equivalent angle measured in - * radians. + * Make DayTimeIntervalType duration from days and hours. * - * @param e - * angle in degrees. A column that evaluates to a double. - * @return - * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that - * evaluates to a double. - * - * @group math_funcs - * @since 2.1.0 - */ - def radians(e: Column): Column = Column.fn("radians", e) - - /** - * Converts an angle measured in degrees to an approximately equivalent angle measured in - * radians. - * - * @param columnName - * angle in degrees. A column that evaluates to a double. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 2.1.0 + * Returns a column that evaluates to an interval. */ - def radians(columnName: String): Column = radians(Column(columnName)) + def make_dt_interval(days: Column, hours: Column): Column = + Column.fn("make_dt_interval", days, hours) /** - * Returns the bucket number into which the value of this expression would fall after being - * evaluated. Note that input arguments must follow conditions listed below; otherwise, the - * method will return null. + * Make DayTimeIntervalType duration from days. * - * @param v - * value to compute a bucket number in the histogram. A column that evaluates to a double or - * interval. - * @param min - * minimum value of the histogram. A column that evaluates to a double or interval. - * @param max - * maximum value of the histogram. A column that evaluates to a double or interval. - * @param numBucket - * the number of buckets. A column that evaluates to a long. - * @return - * the bucket number into which the value would fall after being evaluated. Returns a column - * that evaluates to a long. - * @group math_funcs + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. */ - def width_bucket(v: Column, min: Column, max: Column, numBucket: Column): Column = - Column.fn("width_bucket", v, min, max, numBucket) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Misc functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def make_dt_interval(days: Column): Column = + Column.fn("make_dt_interval", days) /** - * Returns the current catalog. + * Make DayTimeIntervalType duration. * - * @group misc_funcs + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def current_catalog(): Column = Column.fn("current_catalog") + def make_dt_interval(): Column = + Column.fn("make_dt_interval") /** - * Returns the current database. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @group misc_funcs - * @since 3.5.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def current_database(): Column = Column.fn("current_database") + def try_make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days, hours, mins, secs) /** - * Returns the current schema. + * Make interval from years, months, weeks, days, hours, mins and secs. * - * @group misc_funcs + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def current_schema(): Column = Column.fn("current_schema") + def make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_interval", years, months, weeks, days, hours, mins, secs) /** - * Returns the current SQL path as a comma-separated list of qualified schema names. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @group misc_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def current_path(): Column = Column.fn("current_path") + def try_make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days, hours, mins) /** - * Returns the user name of current execution context. + * Make interval from years, months, weeks, days, hours and mins. * - * @group misc_funcs + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def current_user(): Column = Column.fn("current_user") + def make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column): Column = + Column.fn("make_interval", years, months, weeks, days, hours, mins) /** - * Calculates the MD5 digest of a binary column and returns the value as a 32 character hex - * string. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param e - * target column to compute on. A column that evaluates to a binary. - * @group hash_funcs - * @since 1.5.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def md5(e: Column): Column = Column.fn("md5", e) + def try_make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days, hours) /** - * Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex - * string. + * Make interval from years, months, weeks, days and hours. * - * @param e - * target column to compute on. A column that evaluates to a binary. - * @group hash_funcs - * @since 1.5.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def sha1(e: Column): Column = Column.fn("sha1", e) + def make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column): Column = + Column.fn("make_interval", years, months, weeks, days, hours) /** - * Calculates the SHA-2 family of hash functions of a binary column and returns the value as a - * hex string. - * - * @param e - * column to compute SHA-2 on. A column that evaluates to a binary. - * @param numBits - * one of 224, 256, 384, or 512. A column that evaluates to an integer. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @group hash_funcs - * @since 1.5.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def sha2(e: Column, numBits: Int): Column = { - require( - Seq(0, 224, 256, 384, 512).contains(numBits), - s"numBits $numBits is not in the permitted values (0, 224, 256, 384, 512)") - Column.fn("sha2", e, lit(numBits)) - } + def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days) /** - * Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value - * as a bigint. + * Make interval from years, months, weeks and days. * - * @param e - * target column to compute on. A column that evaluates to a binary. - * @group hash_funcs - * @since 1.5.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an interval. */ - def crc32(e: Column): Column = Column.fn("crc32", e) + def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = + Column.fn("make_interval", years, months, weeks, days) /** - * Calculates the hash code of given columns, and returns the result as an int column. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param cols - * one or more columns to compute on. A column of any type. - * @group hash_funcs - * @since 2.0.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an interval. */ - @scala.annotation.varargs - def hash(cols: Column*): Column = Column.fn("hash", cols: _*) + def try_make_interval(years: Column, months: Column, weeks: Column): Column = + Column.fn("try_make_interval", years, months, weeks) /** - * Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, - * and returns the result as a long column. The hash computation uses an initial seed of 42. + * Make interval from years, months and weeks. * - * @param cols - * one or more columns to compute on. A column of any type. - * @group hash_funcs - * @since 3.0.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * The number of weeks, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an interval. */ - @scala.annotation.varargs - def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) + def make_interval(years: Column, months: Column, weeks: Column): Column = + Column.fn("make_interval", years, months, weeks) /** - * Returns a 64-bit hash value of the argument using the XXH3 algorithm. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param col - * the column to hash, which must have string or binary type. - * @group hash_funcs - * @since 4.4.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an interval. */ - def xxh3_64(col: Column): Column = Column.fn("xxh3_64", col) + def try_make_interval(years: Column, months: Column): Column = + Column.fn("try_make_interval", years, months) /** - * Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. + * Make interval from years and months. * - * @param col - * the column to hash, which must have string or binary type. - * @group hash_funcs - * @since 4.4.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def xxh3_128(col: Column): Column = Column.fn("xxh3_128", col) + def make_interval(years: Column, months: Column): Column = + Column.fn("make_interval", years, months) /** - * Returns null if the condition is true, and throws an exception otherwise. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param c - * The condition to check. A column that evaluates to a boolean. - * @group misc_funcs - * @since 3.1.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that always evaluates to NULL. + * Returns a column that evaluates to an interval. */ - def assert_true(c: Column): Column = Column.fn("assert_true", c) + def try_make_interval(years: Column): Column = + Column.fn("try_make_interval", years) /** - * Returns null if the condition is true; throws an exception with the error message otherwise. + * Make interval from years. * - * @param c - * The condition to check. A column that evaluates to a boolean. - * @param e - * The error message to throw. A column that evaluates to a string. - * @group misc_funcs - * @since 3.1.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that always evaluates to NULL. + * Returns a column that evaluates to an interval. */ - def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) + def make_interval(years: Column): Column = + Column.fn("make_interval", years) /** - * Throws an exception with the provided error message. + * Make interval. * - * @param c - * The error message to throw. A column that evaluates to a string. - * @group misc_funcs - * @since 3.1.0 + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that always evaluates to NULL. + * Returns a column that evaluates to an interval. */ - def raise_error(c: Column): Column = Column.fn("raise_error", c) + def make_interval(): Column = + Column.fn("make_interval") /** - * Returns the user name of current execution context. + * Create timestamp from years, months, days, hours, mins, secs and timezone fields. The result + * data type is consistent with the value of configuration `spark.sql.timestampType`. If the + * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. + * Otherwise, it will throw an error instead. * - * @group misc_funcs + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def user(): Column = Column.fn("user") + def make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("make_timestamp", years, months, days, hours, mins, secs, timezone) /** - * Returns the user name of current execution context. + * Create timestamp from years, months, days, hours, mins and secs fields. The result data type + * is consistent with the value of configuration `spark.sql.timestampType`. If the configuration + * `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it + * will throw an error instead. * - * @group misc_funcs - * @since 4.0.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def session_user(): Column = Column.fn("session_user") + def make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_timestamp", years, months, days, hours, mins, secs) /** - * Returns an universally unique identifier (UUID) string. The value is returned as a canonical - * UUID 36-character string. + * Create a local date-time from date, time, and timezone fields. * - * @group misc_funcs - * @since 3.5.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) + def make_timestamp(date: Column, time: Column, timezone: Column): Column = + Column.fn("make_timestamp", date, time, timezone) /** - * Returns an universally unique identifier (UUID) string. The value is returned as a canonical - * UUID 36-character string. + * Create a local date-time from date and time fields. * - * @param seed - * The random number seed to use. A column that evaluates to an integral. Must be a constant. - * @group misc_funcs + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs * @since 4.1.0 * @return - * Returns a column that evaluates to a string. - */ - def uuid(seed: Column): Column = Column.fn("uuid", seed) - - /** - * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the - * given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with `hex` or - * `base64` for a textual value. - * - * @param key - * The secret key, as a binary value. - * @param message - * The message to authenticate, as a binary value. - * @param algorithm - * The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. - * - * @group misc_funcs - * @since 4.3.0 + * Returns a column that evaluates to a timestamp. */ - def hmac(key: Column, message: Column, algorithm: Column): Column = - Column.fn("hmac", key, message, algorithm) + def make_timestamp(date: Column, time: Column): Column = + Column.fn("make_timestamp", date, time) /** - * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and - * SHA-256. The result is returned as raw MAC bytes; wrap it with `hex` or `base64` for a - * textual value. To use a different algorithm, call the three-argument overload. - * - * @param key - * The secret key, as a binary value. - * @param message - * The message to authenticate, as a binary value. + * Try to create a timestamp from years, months, days, hours, mins, secs and timezone fields. + * The result data type is consistent with the value of configuration `spark.sql.timestampType`. + * The function returns NULL on invalid inputs. * - * @group misc_funcs - * @since 4.3.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. */ - def hmac(key: Column, message: Column): Column = - Column.fn("hmac", key, message) + def try_make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("try_make_timestamp", years, months, days, hours, mins, secs, timezone) /** - * Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. - * Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, - * `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization - * vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 - * bytes for GCM. If not provided, a random vector will be generated and prepended to the - * output. Optional additional authenticated data (AAD) is only supported for GCM. If provided - * for encryption, the identical AAD value must be provided for decryption. The default mode is - * GCM. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param iv - * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a - * binary. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. A column that - * evaluates to a binary. + * Try to create a timestamp from years, months, days, hours, mins, and secs fields. The result + * data type is consistent with the value of configuration `spark.sql.timestampType`. The + * function returns NULL on invalid inputs. * - * @group misc_funcs - * @since 3.5.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def aes_encrypt( - input: Column, - key: Column, - mode: Column, - padding: Column, - iv: Column, - aad: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv, aad) + def try_make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_timestamp", years, months, days, hours, mins, secs) /** - * Returns an encrypted value of `input`. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param iv - * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a - * binary. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * Try to create a local date-time from date, time, and timezone fields. * - * @group misc_funcs - * @since 3.5.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = - Column.fn("aes_encrypt", input, key, mode, padding, iv) + def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = + Column.fn("try_make_timestamp", date, time, timezone) /** - * Returns an encrypted value of `input`. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * Try to create a local date-time from date and time fields. * - * @group misc_funcs - * @since 3.5.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = - Column.fn("aes_encrypt", input, key, mode, padding) + def try_make_timestamp(date: Column, time: Column): Column = + Column.fn("try_make_timestamp", date, time) /** - * Returns an encrypted value of `input`. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * Create the current timestamp with local time zone from years, months, days, hours, mins, secs + * and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function + * returns NULL on invalid inputs. Otherwise, it will throw an error instead. * - * @group misc_funcs + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def aes_encrypt(input: Column, key: Column, mode: Column): Column = - Column.fn("aes_encrypt", input, key, mode) + def make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) /** - * Returns an encrypted value of `input`. + * Create the current timestamp with local time zone from years, months, days, hours, mins and + * secs fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns + * NULL on invalid inputs. Otherwise, it will throw an error instead. * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. + */ + def make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs) + + /** + * Try to create the current timestamp with local time zone from years, months, days, hours, + * mins, secs and timezone fields. The function returns NULL on invalid inputs. * - * @group misc_funcs + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. + */ + def try_make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) + + /** + * Try to create the current timestamp with local time zone from years, months, days, hours, + * mins and secs fields. The function returns NULL on invalid inputs. + * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. + */ + def try_make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs) + + /** + * Create local date-time from years, months, days, hours, mins, secs fields. If the + * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. + * Otherwise, it will throw an error instead. + * + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def aes_encrypt(input: Column, key: Column): Column = - Column.fn("aes_encrypt", input, key) + def make_timestamp_ntz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_timestamp_ntz", years, months, days, hours, mins, secs) /** - * Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, - * 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', - * 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is - * only supported for GCM. If provided for encryption, the identical AAD value must be provided - * for decryption. The default mode is GCM. + * Create a local date-time from date and time fields. * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. A column that - * evaluates to a binary. + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. + */ + def make_timestamp_ntz(date: Column, time: Column): Column = + Column.fn("make_timestamp_ntz", date, time) + + /** + * Try to create a local date-time from years, months, days, hours, mins, secs fields. The + * function returns NULL on invalid inputs. * - * @group misc_funcs + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a timestamp. + */ + def try_make_timestamp_ntz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_timestamp_ntz", years, months, days, hours, mins, secs) + + /** + * Try to create a local date-time from date and time fields. + * + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a timestamp. + */ + def try_make_timestamp_ntz(date: Column, time: Column): Column = + Column.fn("try_make_timestamp_ntz", date, time) + + /** + * Make year-month interval from years, months. + * + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def aes_decrypt( - input: Column, - key: Column, - mode: Column, - padding: Column, - aad: Column): Column = - Column.fn("aes_decrypt", input, key, mode, padding, aad) + def make_ym_interval(years: Column, months: Column): Column = + Column.fn("make_ym_interval", years, months) /** - * Returns a decrypted value of `input`. + * Make year-month interval from years. * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to an interval. + */ + def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) + + /** + * Make year-month interval. * - * @group misc_funcs + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = - Column.fn("aes_decrypt", input, key, mode, padding) + def make_ym_interval(): Column = Column.fn("make_ym_interval") + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Hash Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Returns a decrypted value of `input`. + * Calculates the MD5 digest of a binary column and returns the value as a 32 character hex + * string. * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * @param e + * target column to compute on. A column that evaluates to a binary. + * @group hash_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def md5(e: Column): Column = Column.fn("md5", e) + + /** + * Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex + * string. + * + * @param e + * target column to compute on. A column that evaluates to a binary. + * @group hash_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. + */ + def sha1(e: Column): Column = Column.fn("sha1", e) + + /** + * Calculates the SHA-2 family of hash functions of a binary column and returns the value as a + * hex string. + * + * @param e + * column to compute SHA-2 on. A column that evaluates to a binary. + * @param numBits + * one of 224, 256, 384, or 512. A column that evaluates to an integer. * - * @group misc_funcs - * @since 3.5.0 + * @group hash_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def aes_decrypt(input: Column, key: Column, mode: Column): Column = - Column.fn("aes_decrypt", input, key, mode) + def sha2(e: Column, numBits: Int): Column = { + require( + Seq(0, 224, 256, 384, 512).contains(numBits), + s"numBits $numBits is not in the permitted values (0, 224, 256, 384, 512)") + Column.fn("sha2", e, lit(numBits)) + } /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @see - * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value + * as a bigint. * - * @group misc_funcs - * @since 3.5.0 + * @param e + * target column to compute on. A column that evaluates to a binary. + * @group hash_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def aes_decrypt(input: Column, key: Column): Column = - Column.fn("aes_decrypt", input, key) + def crc32(e: Column): Column = Column.fn("crc32", e) /** - * This is a special version of `aes_decrypt` that performs the same operation, but returns a - * NULL value instead of raising an error if the decryption cannot be performed. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. A column that - * evaluates to a binary. + * Calculates the hash code of given columns, and returns the result as an int column. * - * @group misc_funcs - * @since 3.5.0 + * @param cols + * one or more columns to compute on. A column of any type. + * @group hash_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an integer. */ - def try_aes_decrypt( - input: Column, - key: Column, - mode: Column, - padding: Column, - aad: Column): Column = - Column.fn("try_aes_decrypt", input, key, mode, padding, aad) + @scala.annotation.varargs + def hash(cols: Column*): Column = Column.fn("hash", cols: _*) /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, + * and returns the result as a long column. The hash computation uses an initial seed of 42. * - * @group misc_funcs - * @since 3.5.0 + * @param cols + * one or more columns to compute on. A column of any type. + * @group hash_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = - Column.fn("try_aes_decrypt", input, key, mode, padding) + @scala.annotation.varargs + def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * Returns a 64-bit hash value of the argument using the XXH3 algorithm. * - * @group misc_funcs - * @since 3.5.0 + * @param col + * the column to hash, which must have string or binary type. + * @group hash_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = - Column.fn("try_aes_decrypt", input, key, mode) + def xxh3_64(col: Column): Column = Column.fn("xxh3_64", col) /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @see - * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. * - * @group misc_funcs - * @since 3.5.0 + * @param col + * the column to hash, which must have string or binary type. + * @group hash_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def try_aes_decrypt(input: Column, key: Column): Column = - Column.fn("try_aes_decrypt", input, key) + def xxh3_128(col: Column): Column = Column.fn("xxh3_128", col) /** * Returns a sha1 hash value as a hex string of the `col`. @@ -7132,10012 +7303,9916 @@ object functions { */ def sha(col: Column): Column = Column.fn("sha", col) + ////////////////////////////////////////////////////////////////////////////////////////////// + // Collection Functions + ////////////////////////////////////////////////////////////////////////////////////////////// + /** - * Returns the length of the block being read, or -1 if not available. + * Concatenates multiple input columns together into a single column. The function works with + * strings, binary and compatible array columns. * - * @group misc_funcs - * @since 3.5.0 + * @param exprs + * Input columns to concatenate. A column that evaluates to a string, binary or an array. + * @note + * Returns null if any of the input columns are null. + * + * @group collection_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def input_file_block_length(): Column = Column.fn("input_file_block_length") + @scala.annotation.varargs + def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) /** - * Returns the start offset of the block being read, or -1 if not available. + * Returns element of array at given index in value if column is array. Returns value for the + * given key in value if column is map. * - * @group misc_funcs - * @since 3.5.0 + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. + * @group collection_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the element type of the input array, or the value type of the input + * map. */ - def input_file_block_start(): Column = Column.fn("input_file_block_start") + def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) /** - * Calls a method with reflection. + * (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will + * throw an error. If index < 0, accesses elements from the last to the first. The function + * always returns NULL if the index exceeds the length of the array. * - * @group misc_funcs + * (map, key) - Returns value for given key. The function always returns NULL if the key is not + * contained in the map. + * + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. + * @group collection_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the element type of the input array, or the value type of the input + * map. */ - @scala.annotation.varargs - def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) + def try_element_at(column: Column, value: Column): Column = + Column.fn("try_element_at", column, value) /** - * Calls a method with reflection. + * Sorts the input array in ascending order. Null elements will be placed at the end of the + * returned array. NaN is greater than any non-NaN elements for double/float type. * - * @group misc_funcs - * @since 3.5.0 + * The elements of the input array must be orderable. For example, when the array elements are + * structs, the default comparator compares the struct fields in schema order. Therefore, all + * fields in the struct must be orderable. If the default comparator does not support the input + * type, you can specify a custom comparator. + * + * @param e + * The array to sort. A column that evaluates to an array. + * @group collection_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - @scala.annotation.varargs - def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) + def array_sort(e: Column): Column = Column.fn("array_sort", e) + + /** + * Sorts the input array based on the given comparator function. The comparator will take two + * arguments representing two elements of the array. It returns a negative integer, 0, or a + * positive integer as the first element is less than, equal to, or greater than the second + * element. If the comparator function returns null, the function will fail and raise an error. + * + * @param e + * The array to sort. A column that evaluates to an array. + * @param comparator + * A binary comparator function that returns a negative integer, 0, or a positive integer as + * the first element is less than, equal to, or greater than the second element. + * @group collection_funcs + * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. + */ + def array_sort(e: Column, comparator: (Column, Column) => Column): Column = + Column.fn("array_sort", e, createLambda(comparator)) + + private def createLambda(f: Column => Column) = { + val x = internal.UnresolvedNamedLambdaVariable("x") + val function = f(Column(x)).node + Column(internal.LambdaFunction(function, Seq(x))) + } + + private def createLambda(f: (Column, Column) => Column) = { + val x = internal.UnresolvedNamedLambdaVariable("x") + val y = internal.UnresolvedNamedLambdaVariable("y") + val function = f(Column(x), Column(y)).node + Column(internal.LambdaFunction(function, Seq(x, y))) + } + + private def createLambda(f: (Column, Column, Column) => Column) = { + val x = internal.UnresolvedNamedLambdaVariable("x") + val y = internal.UnresolvedNamedLambdaVariable("y") + val z = internal.UnresolvedNamedLambdaVariable("z") + val function = f(Column(x), Column(y), Column(z)).node + Column(internal.LambdaFunction(function, Seq(x, y, z))) + } /** - * This is a special version of `reflect` that performs the same operation, but returns a NULL - * value instead of raising an error if the invoke method thrown exception. + * Returns an array of elements after applying a transformation to each element in the input + * array. + * {{{ + * df.select(transform(col("i"), x => x + 1)) + * }}} * - * @group misc_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a string. - */ - @scala.annotation.varargs - def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) - - /** - * Returns the Spark version. The string contains 2 fields, the first being a release version - * and the second being a git revision. + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => transformed_col, the lambda function to transform the input column. * - * @group misc_funcs - * @since 3.5.0 + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def version(): Column = Column.fn("version") + def transform(column: Column, f: Column => Column): Column = + Column.fn("transform", column, createLambda(f)) /** - * Return DDL-formatted type string for the data type of the input. + * Returns an array of elements after applying a transformation to each element in the input + * array. + * {{{ + * df.select(transform(col("i"), (x, i) => x + i)) + * }}} * - * @param col - * The value whose data type is returned. A column of any type. - * @group misc_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a string. - */ - def typeof(col: Column): Column = Column.fn("typeof", col) - - /** - * Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default - * unless specified otherwise. + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * (col, index) => transformed_col, the lambda function to transform the input column given + * the index. Indices start at 0. * - * @param cols - * The first column must be a constant integer for the number of rows, and the remaining - * columns are the input elements to be separated into rows. - * @group generator_funcs - * @since 3.5.0 + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - @scala.annotation.varargs - def stack(cols: Column*): Column = Column.fn("stack", cols: _*) + def transform(column: Column, f: (Column, Column) => Column): Column = + Column.fn("transform", column, createLambda(f)) /** - * Returns a random value with independent and identically distributed (i.i.d.) values with the - * specified range of numbers. The provided numbers specifying the minimum and maximum values of - * the range must be constant. If both of these numbers are integers, then the result will also - * be an integer. Otherwise if one or both of these are floating-point numbers, then the result - * will also be a floating-point number. + * Returns whether a predicate holds for one or more elements in the array. + * {{{ + * df.select(exists(col("i"), _ % 2 === 0)) + * }}} * - * @param min - * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. - * @param max - * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. - * @group math_funcs - * @since 4.0.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to check the input column. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def uniform(min: Column, max: Column): Column = - uniform(min, max, lit(SparkClassUtils.random.nextLong)) + def exists(column: Column, f: Column => Column): Column = + Column.fn("exists", column, createLambda(f)) /** - * Returns a random value with independent and identically distributed (i.i.d.) values with the - * specified range of numbers, with the chosen random seed. The provided numbers specifying the - * minimum and maximum values of the range must be constant. If both of these numbers are - * integers, then the result will also be an integer. Otherwise if one or both of these are - * floating-point numbers, then the result will also be a floating-point number. + * Returns whether a predicate holds for every element in the array. + * {{{ + * df.select(forall(col("i"), x => x % 2 === 0)) + * }}} * - * @param min - * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. - * @param max - * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. - * @param seed - * Random number seed to use. A column that evaluates to an integral. Must be a constant. - * @group math_funcs - * @since 4.0.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to check the input column. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def uniform(min: Column, max: Column, seed: Column): Column = - Column.fn("uniform", min, max, seed) + def forall(column: Column, f: Column => Column): Column = + Column.fn("forall", column, createLambda(f)) /** - * Returns a random value with independent and identically distributed (i.i.d.) uniformly - * distributed values in [0, 1). + * Returns an array of elements for which a predicate holds in a given array. + * {{{ + * df.select(filter(col("s"), x => x % 2 === 0)) + * }}} * - * @param seed - * Random number seed to use. A column that evaluates to an integral. Must be a constant. - * @group math_funcs - * @since 3.5.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to filter the input column. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def random(seed: Column): Column = Column.fn("random", seed) + def filter(column: Column, f: Column => Column): Column = + Column.fn("filter", column, createLambda(f)) /** - * Returns a random value with independent and identically distributed (i.i.d.) uniformly - * distributed values in [0, 1). + * Returns an array of elements for which a predicate holds in a given array. + * {{{ + * df.select(filter(col("s"), (x, i) => i % 2 === 0)) + * }}} * - * @group math_funcs - * @since 3.5.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * (col, index) => predicate, the Boolean predicate to filter the input column given the + * index. Indices start at 0. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def random(): Column = random(lit(SparkClassUtils.random.nextLong)) + def filter(column: Column, f: (Column, Column) => Column): Column = + Column.fn("filter", column, createLambda(f)) /** - * Returns the bit position for the given input column. + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. The final state is converted into the final result by applying a finish + * function. + * {{{ + * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) + * }}} * - * @param col - * The input column. A column that evaluates to an integral. - * @group misc_funcs - * @since 3.5.0 + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value. + * @param finish + * combined_value => final_value, the lambda function to convert the combined value of all + * inputs to final result. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the initial value. */ - def bitmap_bit_position(col: Column): Column = - Column.fn("bitmap_bit_position", col) + def aggregate( + expr: Column, + initialValue: Column, + merge: (Column, Column) => Column, + finish: Column => Column): Column = + Column.fn("aggregate", expr, initialValue, createLambda(merge), createLambda(finish)) /** - * Returns the bucket number for the given input column. + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. + * {{{ + * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x)) + * }}} * - * @param col - * The input column. A column that evaluates to an integral. - * @group misc_funcs - * @since 3.5.0 + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the initial value. */ - def bitmap_bucket_number(col: Column): Column = - Column.fn("bitmap_bucket_number", col) + def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = + aggregate(expr, initialValue, merge, c => c) /** - * Returns a bitmap with the positions of the bits set from all the values from the input - * column. The input column will most likely be bitmap_bit_position(). + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. The final state is converted into the final result by applying a finish + * function. + * {{{ + * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) + * }}} * - * @param col - * The input column will most likely be bitmap_bit_position(). A column that evaluates to an - * integral. - * @group agg_funcs + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value. + * @param finish + * combined_value => final_value, the lambda function to convert the combined value of all + * inputs to final result. + * + * @group collection_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the initial value. */ - def bitmap_construct_agg(col: Column): Column = - Column.fn("bitmap_construct_agg", col) + def reduce( + expr: Column, + initialValue: Column, + merge: (Column, Column) => Column, + finish: Column => Column): Column = + Column.fn("reduce", expr, initialValue, createLambda(merge), createLambda(finish)) /** - * Returns the number of set bits in the input bitmap. + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. + * {{{ + * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x)) + * }}} * - * @param col - * The input bitmap. A column that evaluates to a binary. - * @group misc_funcs + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value + * @group collection_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the initial value. */ - def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) + def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = + reduce(expr, initialValue, merge, c => c) /** - * Returns a bitmap that is the bitwise AND of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * Merge two given arrays, element-wise, into a single array using a function. If one array is + * shorter, nulls are appended at the end to match the length of the longer array, before + * applying the function. + * {{{ + * df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y)) + * }}} * * @param left - * A column that evaluates to a binary bitmap. + * the left input array column. A column that evaluates to an array. * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 - * @return - * Returns a column that evaluates to a binary bitmap. - */ - def bitmap_and(left: Column, right: Column): Column = Column.fn("bitmap_and", left, right) - - /** - * Returns a bitmap that is the bitwise OR of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * the right input array column. A column that evaluates to an array. + * @param f + * (lCol, rCol) => col, the lambda function to merge two input columns into one column. * - * @param left - * A column that evaluates to a binary bitmap. - * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary bitmap. + * Returns a column that evaluates to an array. */ - def bitmap_or(left: Column, right: Column): Column = Column.fn("bitmap_or", left, right) + def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = + Column.fn("zip_with", left, right, createLambda(f)) /** - * Returns a bitmap that is the bitwise AND NOT of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * Applies a function to every key-value pair in a map and returns a map with the results of + * those applications as the new keys for the pairs. + * {{{ + * df.select(transform_keys(col("i"), (k, v) => k + v)) + * }}} * - * @param left - * A column that evaluates to a binary bitmap. - * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 - * @return - * Returns a column that evaluates to a binary bitmap. - */ - def bitmap_andnot(left: Column, right: Column): Column = - Column.fn("bitmap_andnot", left, right) - - /** - * Returns a bitmap that is the bitwise XOR of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * @param expr + * the input map column. A column that evaluates to a map. + * @param f + * (key, value) => new_key, the lambda function to transform the key of input map column * - * @param left - * A column that evaluates to a binary bitmap. - * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary bitmap. + * Returns a column that evaluates to a map. */ - def bitmap_xor(left: Column, right: Column): Column = Column.fn("bitmap_xor", left, right) + def transform_keys(expr: Column, f: (Column, Column) => Column): Column = + Column.fn("transform_keys", expr, createLambda(f)) /** - * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. The - * input column should be bitmaps created from bitmap_construct_agg(). + * Applies a function to every key-value pair in a map and returns a map with the results of + * those applications as the new values for the pairs. + * {{{ + * df.select(transform_values(col("i"), (k, v) => k + v)) + * }}} * - * @param col - * The input column should be bitmaps created from bitmap_construct_agg(). A column that - * evaluates to a binary. - * @group agg_funcs - * @since 3.5.0 + * @param expr + * the input map column. A column that evaluates to a map. + * @param f + * (key, value) => new_value, the lambda function to transform the value of input map column + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a map. */ - def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) + def transform_values(expr: Column, f: (Column, Column) => Column): Column = + Column.fn("transform_values", expr, createLambda(f)) /** - * Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. The - * input column should be bitmaps created from bitmap_construct_agg(). + * Returns a map whose key-value pairs satisfy a predicate. + * {{{ + * df.select(map_filter(col("m"), (k, v) => k * 10 === v)) + * }}} * - * @param col - * The input column should be bitmaps created from bitmap_construct_agg(). A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.1.0 + * @param expr + * the input map column. A column that evaluates to a map. + * @param f + * (key, value) => predicate, the Boolean predicate to filter the input map column + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a map. */ - def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) + def map_filter(expr: Column, f: (Column, Column) => Column): Column = + Column.fn("map_filter", expr, createLambda(f)) /** - * Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. The - * input column should be bitmaps created from bitmap_construct_agg(). + * Merge two given maps, key-wise into a single map using a function. + * {{{ + * df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)) + * }}} * - * @param col - * A column containing bitmaps created by bitmap_construct_agg() and evaluating to binary - * data. - * @group agg_funcs - * @since 4.4.0 + * @param left + * the left input map column. A column that evaluates to a map. + * @param right + * the right input map column. A column that evaluates to a map. + * @param f + * (key, value1, value2) => new_value, the lambda function to merge the map values + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a map. */ - def bitmap_xor_agg(col: Column): Column = Column.fn("bitmap_xor_agg", col) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // String functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = + Column.fn("map_zip_with", left, right, createLambda(f)) /** - * Computes the numeric value of the first character of the string column, and returns the - * result as an int column. + * Returns length of array or map. + * + * This function returns -1 for null input only if spark.sql.ansi.enabled is false and + * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the + * default settings, the function returns null for null input. * * @param e - * The target column to work on. A column that evaluates to a string. - * @group string_funcs + * the target column. A column that evaluates to an array or a map. + * @group collection_funcs * @since 1.5.0 * @return * Returns a column that evaluates to an integer. */ - def ascii(e: Column): Column = Column.fn("ascii", e) + def size(e: Column): Column = Column.fn("size", e) /** - * Computes the BASE64 encoding of a binary column and returns it as a string column. This is - * the reverse of unbase64. + * Returns length of array or map. This is an alias of `size` function. * - * @param e - * The target column to work on. A column that evaluates to a binary. - * @group string_funcs - * @since 1.5.0 - * @return - * Returns a column that evaluates to a string. - */ - def base64(e: Column): Column = Column.fn("base64", e) - - /** - * Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a string column. - * This is the reverse of from_base32. + * This function returns -1 for null input only if spark.sql.ansi.enabled is false and + * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the + * default settings, the function returns null for null input. * * @param e - * The target column to work on. A column that evaluates to a binary. - * @group string_funcs - * @since 4.3.0 + * the target column. A column that evaluates to an array or a map. + * @group collection_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an integer. */ - def to_base32(e: Column): Column = Column.fn("to_base32", e) + def cardinality(e: Column): Column = Column.fn("cardinality", e) /** - * Calculates the bit length for the specified string column. - * - * @param e - * The source column or strings. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.3.0 + * Returns a reversed string or an array with reverse order of elements. + * @param e + * the input column. A column that evaluates to a string, a binary, or an array. + * @group collection_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def bit_length(e: Column): Column = Column.fn("bit_length", e) + def reverse(e: Column): Column = Column.fn("reverse", e) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Array Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Concatenates multiple input string columns together into a single string column, using the - * given separator. - * - * @param sep - * The words separator. A column that evaluates to a string. Must be a constant. - * @param exprs - * The list of columns to work on. Each a column that evaluates to a string or an array of - * strings. - * @note - * Input strings which are null are skipped. + * Creates a new array column. The input columns must all have the same data type. * - * @group string_funcs - * @since 1.5.0 + * @param cols + * The columns to combine into an array. Each is a column of any type, and all must share the + * same data type. + * @group array_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ @scala.annotation.varargs - def concat_ws(sep: String, exprs: Column*): Column = - Column.fn("concat_ws", lit(sep) +: exprs: _*) + def array(cols: Column*): Column = Column.fn("array", cols: _*) /** - * Computes the first argument into a string from a binary using the provided character set (one - * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either - * argument is null, the result will also be null. + * Creates a new array column. The input columns must all have the same data type. * - * @param value - * The target column to work on. A column that evaluates to a binary. - * @param charset - * The charset to use to decode to. A column that evaluates to a string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @group array_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def decode(value: Column, charset: String): Column = - Column.fn("decode", value, lit(charset)) + @scala.annotation.varargs + def array(colName: String, colNames: String*): Column = { + array((colName +: colNames).map(col): _*) + } /** - * Computes the first argument into a binary from a string using the provided character set (one - * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either - * argument is null, the result will also be null. - * + * Returns true if the array contains `value`, false if not. Returns null if the array or + * `value` is null, or if `value` is not found and the array contains a null element. + * @param column + * the target column containing the arrays. A column that evaluates to an array. * @param value - * The target column to work on. A column that evaluates to a string. - * @param charset - * The charset to use to encode. A column that evaluates to a string. Must be a constant. - * @group string_funcs + * the value to check for in the array. A column that evaluates to a value matching the + * array's element type. + * @group array_funcs * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def encode(value: Column, charset: String): Column = - Column.fn("encode", value, lit(charset)) + def array_contains(column: Column, value: Any): Column = + Column.fn("array_contains", column, lit(value)) /** - * Returns true if the input is a valid UTF-8 string, otherwise returns false. + * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. + * The new element/column is located at end of the ARRAY. * - * @param str - * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a - * string. - * @group string_funcs - * @since 4.0.0 + * @param column + * the source column containing the array. A column that evaluates to an array. + * @param element + * the value to append to the array. A column that evaluates to a value matching the array's + * element type. + * @group array_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def is_valid_utf8(str: Column): Column = - Column.fn("is_valid_utf8", str) + def array_append(column: Column, element: Any): Column = + Column.fn("array_append", column, lit(element)) /** - * Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the - * Unicode replacement character (U+FFFD). - * - * @param str - * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a - * string. - * @group string_funcs - * @since 4.0.0 + * Returns `true` if `a1` and `a2` have at least one non-null element in common. If not and both + * the arrays are non-empty and any of them contains a `null`, it returns `null`. It returns + * `false` otherwise. + * @param a1 + * the first input array. A column that evaluates to an array. + * @param a2 + * the second input array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a boolean. */ - def make_valid_utf8(str: Column): Column = - Column.fn("make_valid_utf8", str) + def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) /** - * Returns the input value if it corresponds to a valid UTF-8 string, or emits a - * SparkIllegalArgumentException exception otherwise. + * Returns an array containing all the elements in `x` from index `start` (or starting from the + * end if `start` is negative) with the specified `length`. * - * @param str - * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a - * string. - * @group string_funcs - * @since 4.0.0 + * @param x + * the array column to be sliced. A column that evaluates to an array. + * @param start + * the starting index. A column that evaluates to an integer. + * @param length + * the length of the slice. A column that evaluates to an integer. + * + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def validate_utf8(str: Column): Column = - Column.fn("validate_utf8", str) + def slice(x: Column, start: Int, length: Int): Column = + slice(x, lit(start), lit(length)) /** - * Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. + * Returns an array containing all the elements in `x` from index `start` (or starting from the + * end if `start` is negative) with the specified `length`. * - * @param str - * the input value. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * @param x + * the array column to be sliced. A column that evaluates to an array. + * @param start + * the starting index. A column that evaluates to an integer. + * @param length + * the length of the slice. A column that evaluates to an integer. + * + * @group array_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def try_validate_utf8(str: Column): Column = - Column.fn("try_validate_utf8", str) + def slice(x: Column, start: Column, length: Column): Column = + Column.fn("slice", x, start, length) /** - * Returns the Unicode normalization of `str` using the given normalization `form`. Valid forms - * are 'NFC', 'NFD', 'NFKC', and 'NFKD', as defined by Unicode Standard Annex #15. The form name - * is case-insensitive. Normalization is backed by Spark's bundled ICU4J library rather than the - * JVM's own Unicode data, so results are stable across JVM vendors and versions. + * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is + * negative or greater than the number of elements in the array. * - * @param str - * the input string to normalize. - * @param form - * the normalization form: 'NFC', 'NFD', 'NFKC', or 'NFKD'. - * @group string_funcs - * @since 4.4.0 - */ - def normalize(str: Column, form: Column): Column = - Column.fn("normalize", str, form) - - /** - * Returns the Unicode normalization of `str` using the default form 'NFC'. To use a different - * form, call the two-argument overload. + * @param x + * the array column to be trimmed. A column that evaluates to an array. + * @param n + * the number of elements to remove from the end of the array. Must be between 0 and the + * number of elements in the array (inclusive). * - * @param str - * the input string to normalize. - * @group string_funcs + * @group array_funcs * @since 4.4.0 + * @return + * Returns a column that evaluates to an array. */ - def normalize(str: Column): Column = - Column.fn("normalize", str) + def trim_array(x: Column, n: Int): Column = trim_array(x, lit(n)) /** - * Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places with - * HALF_EVEN round mode, and returns the result as a string column. - * - * If d is 0, the result has no decimal point or fractional part. If d is less than 0, the - * result will be null. + * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is + * negative or greater than the number of elements in the array. * * @param x - * the numeric value to be formatted. A column that evaluates to a numeric. - * @param d - * the number of decimal places. A column that evaluates to an integral. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * the array column to be trimmed. A column that evaluates to an array. + * @param n + * the number of elements to remove from the end of the array. Must be between 0 and the + * number of elements in the array (inclusive). + * + * @group array_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) + def trim_array(x: Column, n: Column): Column = Column.fn("trim_array", x, n) /** - * Formats the arguments in printf-style and returns the result as a string column. - * - * @param format - * the format string that can contain embedded format tags. A column that evaluates to a - * string. Must be a constant. - * @param arguments - * the values to be used in formatting. Columns that evaluate to any type. - * @group string_funcs - * @since 1.5.0 + * Concatenates the elements of `column` using the `delimiter`. Null values are replaced with + * `nullReplacement`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. + * @param nullReplacement + * the string used to replace null values. A column that evaluates to a string. + * @group array_funcs + * @since 2.4.0 * @return * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def format_string(format: String, arguments: Column*): Column = - Column.fn("format_string", lit(format) +: arguments: _*) + def array_join(column: Column, delimiter: String, nullReplacement: String): Column = + Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) /** - * Returns a new string column by converting the first letter of each word to uppercase. Words - * are delimited by whitespace. - * - * For example, "hello world" will become "Hello World". - * - * @param e - * the target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * Concatenates the elements of `column` using the `delimiter`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. + * @group array_funcs + * @since 2.4.0 * @return * Returns a column that evaluates to a string. */ - def initcap(e: Column): Column = Column.fn("initcap", e) + def array_join(column: Column, delimiter: String): Column = + Column.fn("array_join", column, lit(delimiter)) /** - * Locate the position of the first occurrence of substr column in the given string. Returns + * Locates the position of the first occurrence of the value in the given array as long. Returns * null if either of the arguments are null. * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. Must be a constant. + * @param column + * The array to search. A column that evaluates to an array. + * @param value + * The value to locate. A column. * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. + * The position is not zero based, but 1 based index. Returns 0 if value could not be found in + * array. * - * @group string_funcs - * @since 1.5.0 + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a long. */ - def instr(str: Column, substring: String): Column = instr(str, lit(substring)) + def array_position(column: Column, value: Any): Column = + Column.fn("array_position", column, lit(value)) /** - * Locate the position of the first occurrence of substr column in the given string. Returns - * null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. + * Returns element of array at given (0-based) index. If the index points outside of the array + * boundaries, then this function returns NULL. * - * @group string_funcs - * @since 4.0.0 + * @param column + * The array to extract from. A column that evaluates to an array. + * @param index + * The 0-based index. A column that evaluates to an integral. + * @group array_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the element type of the input array. */ - def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) + def get(column: Column, index: Column): Column = Column.fn("get", column, index) /** - * Locate the position of the first occurrence of `substring` in `str`, starting the search from - * position `start`. Returns null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. Must be a - * constant. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. + * Remove all elements that equal to element from the given array. * - * @group string_funcs - * @since 4.3.0 + * @param column + * The array to remove from. A column that evaluates to an array. + * @param element + * The element to remove. A column. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def instr(str: Column, substring: Column, start: Int): Column = - Column.fn("instr", str, substring, lit(start)) + def array_remove(column: Column, element: Any): Column = + Column.fn("array_remove", column, lit(element)) /** - * Locate the position of the first occurrence of `substring` in `str`, starting the search from - * position `start`. Returns null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. + * Remove all null elements from the given array. * - * @group string_funcs - * @since 4.3.0 + * @param column + * The array to compact. A column that evaluates to an array. + * @group array_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def instr(str: Column, substring: Column, start: Column): Column = - Column.fn("instr", str, substring, start) + def array_compact(column: Column): Column = Column.fn("array_compact", column) /** - * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the - * search from position `start`. Returns null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. Must be a - * constant. - * @param occurrence - * which occurrence of the substring to locate. A column that evaluates to an integral. Must - * be a constant. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. - * @note - * The `occurrence` parameter must be a positive integer. + * Returns an array containing value as well as all elements from array. The new element is + * positioned at the beginning of the array. * - * @group string_funcs - * @since 4.3.0 + * @param column + * The array to prepend to. A column that evaluates to an array. + * @param element + * The element to prepend. A column. + * @group array_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = - Column.fn("instr", str, substring, lit(start), lit(occurrence)) + def array_prepend(column: Column, element: Any): Column = + Column.fn("array_prepend", column, lit(element)) /** - * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the - * search from position `start`. Returns null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. - * @param occurrence - * which occurrence of the substring to locate. A column that evaluates to an integral. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. - * @note - * The `occurrence` parameter must be a positive integer. - * - * @group string_funcs - * @since 4.3.0 + * Removes duplicate values from the array. + * @param e + * The array to deduplicate. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = - Column.fn("instr", str, substring, start, occurrence) + def array_distinct(e: Column): Column = Column.fn("array_distinct", e) /** - * Computes the character length of a given string or number of bytes of a binary string. The - * length of character strings include the trailing spaces. The length of binary strings - * includes binary zeros. + * Returns an array of the elements in the intersection of the given two arrays, without + * duplicates. * - * @param e - * the target column to work on. A column that evaluates to a string or binary. - * @group string_funcs - * @since 1.5.0 + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def length(e: Column): Column = Column.fn("length", e) + def array_intersect(col1: Column, col2: Column): Column = + Column.fn("array_intersect", col1, col2) /** - * Computes the character length of a given string or number of bytes of a binary string. The - * length of character strings include the trailing spaces. The length of binary strings - * includes binary zeros. + * Adds an item into a given array at a specified position * - * @param e - * the target column to work on. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @param arr + * The array to insert into. A column that evaluates to an array. + * @param pos + * The 1-based position at which to insert (negative counts from the end). A column that + * evaluates to an integral. + * @param value + * The value to insert. A column. + * @group array_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def len(e: Column): Column = Column.fn("len", e) + def array_insert(arr: Column, pos: Column, value: Column): Column = + Column.fn("array_insert", arr, pos, value) /** - * Converts a string column to lower case. + * Returns an array of the elements in the union of the given two arrays, without duplicates. * - * @param e - * the target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.3.0 + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def lower(e: Column): Column = Column.fn("lower", e) + def array_union(col1: Column, col2: Column): Column = + Column.fn("array_union", col1, col2) /** - * Computes the Levenshtein distance of the two given string columns if it's less than or equal - * to a given threshold. - * @param l - * the first input column. A column that evaluates to a string. - * @param r - * the second input column. A column that evaluates to a string. - * @param threshold - * the maximum distance to compute. A column that evaluates to an integral. Must be a - * constant. + * Returns an array of the elements in the first array but not in the second array, without + * duplicates. The order of elements in the result is not determined + * + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * result distance, or -1. Returns a column that evaluates to an integer. - * @group string_funcs - * @since 3.5.0 + * Returns a column that evaluates to an array. */ - def levenshtein(l: Column, r: Column, threshold: Int): Column = - Column.fn("levenshtein", l, r, lit(threshold)) + def array_except(col1: Column, col2: Column): Column = + Column.fn("array_except", col1, col2) /** - * Computes the Levenshtein distance of the two given string columns. - * @param l - * the first input column. A column that evaluates to a string. - * @param r - * the second input column. A column that evaluates to a string. - * @group string_funcs + * Sorts the input array for the given column in ascending order, according to the natural + * ordering of the array elements. Null elements will be placed at the beginning of the returned + * array. + * + * @param e + * the array column to sort. A column that evaluates to an array. + * @group array_funcs * @since 1.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) + def sort_array(e: Column): Column = sort_array(e, asc = true) /** - * Computes the Jaro-Winkler similarity between the two given string columns. The result is a - * double between 0.0 (no similarity) and 1.0 (identical). - * @param l - * A column that evaluates to a string. - * @param r - * A column that evaluates to a string. - * @group string_funcs - * @since 4.3.0 + * Sorts the input array for the given column in ascending or descending order, according to the + * natural ordering of the array elements. NaN is greater than any non-NaN elements for + * double/float type. Null elements will be placed at the beginning of the returned array in + * ascending order or at the end of the returned array in descending order. + * + * @param e + * the array column to sort. A column that evaluates to an array. + * @param asc + * whether to sort in ascending order. A column that evaluates to a boolean. Must be a + * constant. + * @group array_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def jaro_winkler_similarity(l: Column, r: Column): Column = - Column.fn("jaro_winkler_similarity", l, r) + def sort_array(e: Column, asc: Boolean): Column = Column.fn("sort_array", e, lit(asc)) /** - * Locate the position of the first occurrence of substr. - * - * @param substr - * The substring to find. A column that evaluates to a string. - * @param str - * A column that evaluates to a string. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. + * Returns the minimum value in the array. NaN is greater than any non-NaN elements for + * double/float type. NULL elements are skipped. * - * @group string_funcs - * @since 1.5.0 + * @param e + * the array column. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the element type of the input array. */ - def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) + def array_min(e: Column): Column = Column.fn("array_min", e) /** - * Locate the position of the first occurrence of substr in a string column, after position pos. - * - * @param substr - * The substring to find. A column that evaluates to a string. - * @param str - * A column that evaluates to a string. - * @param pos - * The starting position. A column that evaluates to an integer. - * @note - * The position is not zero based, but 1 based index. returns 0 if substr could not be found - * in str. + * Returns the maximum value in the array. NaN is greater than any non-NaN elements for + * double/float type. NULL elements are skipped. * - * @group string_funcs - * @since 1.5.0 + * @param e + * the input column. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the element type of the input array. */ - def locate(substr: String, str: Column, pos: Int): Column = - Column.fn("locate", lit(substr), str, lit(pos)) + def array_max(e: Column): Column = Column.fn("array_max", e) /** - * Left-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. + * Returns the total number of elements in the array. The function returns null for null input. * - * @param str - * A column that evaluates to a string. - * @param len - * The length of the padded result. A column that evaluates to an integer. - * @param pad - * The padding string. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * @param e + * the input column. A column that evaluates to an array. + * @group array_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) + def array_size(e: Column): Column = Column.fn("array_size", e) /** - * Left-pad the binary column with pad to a byte length of len. If the binary column is longer - * than len, the return value is shortened to len bytes. + * Returns a random permutation of the given array. * - * @param str - * A column that evaluates to a binary. - * @param len - * The byte length of the padded result. A column that evaluates to an integer. - * @param pad - * The padding bytes. A column that evaluates to a binary. - * @group string_funcs - * @since 3.3.0 + * @param e + * the input column. A column that evaluates to an array. + * @note + * The function is non-deterministic. + * + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) + def shuffle(e: Column): Column = shuffle(e, lit(SparkClassUtils.random.nextLong)) /** - * Left-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. + * Returns a random permutation of the given array. * - * @param str - * A column that evaluates to a string. - * @param len - * The length of the padded result. A column that evaluates to an integer. - * @param pad - * The padding string. A column that evaluates to a string. - * @group string_funcs + * @param e + * the input column. A column that evaluates to an array. + * @param seed + * the seed for the random generator. A column that evaluates to an integral. Must be a + * constant. + * @note + * The function is non-deterministic. + * + * @group array_funcs * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) + def shuffle(e: Column, seed: Column): Column = Column.fn("shuffle", e, seed) /** - * Trim the spaces from left end for the specified string value. - * + * Creates a single array from an array of arrays. If a structure of nested arrays is deeper + * than two levels, only one level of nesting is removed. * @param e - * A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * the input column. A column that evaluates to an array of arrays. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def ltrim(e: Column): Column = Column.fn("ltrim", e) + def flatten(e: Column): Column = Column.fn("flatten", e) /** - * Trim the specified character string from left end for the specified string column. - * @param e - * A column that evaluates to a string. - * @param trimString - * The trim string. A column that evaluates to a string. - * @group string_funcs - * @since 2.3.0 + * Generate a sequence of integers from start to stop, incrementing by step. + * + * @param start + * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a + * date, or a timestamp. + * @param stop + * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, + * or a timestamp. + * @param step + * the value to add to the current element to get the next element. A column that evaluates to + * an integral or interval. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) + def sequence(start: Column, stop: Column, step: Column): Column = + Column.fn("sequence", start, stop, step) /** - * Trim the specified character string from left end for the specified string column. - * @param e - * A column that evaluates to a string. - * @param trim - * The trim string. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * Generate a sequence of integers from start to stop, incrementing by 1 if start is less than + * or equal to stop, otherwise -1. + * + * @param start + * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a + * date, or a timestamp. + * @param stop + * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, + * or a timestamp. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) + def sequence(start: Column, stop: Column): Column = Column.fn("sequence", start, stop) /** - * Calculates the byte length for the specified string column. + * Creates an array containing the left argument repeated the number of times given by the right + * argument. * - * @param e - * A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.3.0 + * @param left + * the value to repeat. A column that evaluates to any type. + * @param right + * the number of times to repeat the value. A column that evaluates to an integral. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def octet_length(e: Column): Column = Column.fn("octet_length", e) + def array_repeat(left: Column, right: Column): Column = Column.fn("array_repeat", left, right) /** - * Marks a given column with specified collation. + * Creates an array containing the left argument repeated the number of times given by the right + * argument. * * @param e - * A column that evaluates to a string. - * @param collation - * The collation name. A column that evaluates to a string. Must be a constant. - * @group string_funcs - * @since 4.0.0 + * the value to repeat. A column that evaluates to any type. + * @param count + * the number of times to repeat the value. A column that evaluates to an integral. Must be a + * constant. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) + def array_repeat(e: Column, count: Int): Column = array_repeat(e, lit(count)) /** - * Returns the collation name of a given column. - * + * Returns a merged array of structs in which the N-th struct contains all N-th values of input + * arrays. * @param e - * A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * the columns of arrays to be merged. Each is a column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def collation(e: Column): Column = Column.fn("collation", e) + @scala.annotation.varargs + def arrays_zip(e: Column*): Column = Column.fn("arrays_zip", e: _*) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Struct Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Returns true if `str` matches `regexp`, or false otherwise. + * Creates a struct with the given field names and values. * - * @param str - * A column that evaluates to a string. - * @param regexp - * The regular expression pattern. A column that evaluates to a string. - * @group predicate_funcs + * @param cols + * The field names and values grouped as pairs (name1, value1, name2, value2, ...). Names are + * columns that evaluate to a string; values are columns of any type. + * @group struct_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a struct. */ - def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) + @scala.annotation.varargs + def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) /** - * Returns true if `str` matches `regexp`, or false otherwise. + * Creates a new struct column. If the input column is a column in a `DataFrame`, or a derived + * column expression that is named (i.e. aliased), its name would be retained as the + * StructField's name, otherwise, the newly generated StructField's name would be auto generated + * as `col` with a suffix `index + 1`, i.e. col1, col2, col3, ... * - * @param str - * A column that evaluates to a string. - * @param regexp - * The regular expression pattern. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * @param cols + * the columns to contain in the output struct. A column of any type. + * @group struct_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a struct. */ - def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) + @scala.annotation.varargs + def struct(cols: Column*): Column = Column.fn("struct", cols: _*) /** - * Returns true if `str` matches `regexp`, or false otherwise. + * Creates a new struct column that composes multiple input columns. * - * @param str - * A column that evaluates to a string. - * @param regexp - * The regular expression pattern. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * @param colName + * the name of the first column to contain in the output struct. + * @param colNames + * the names of the remaining columns to contain in the output struct. + * @group struct_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a struct. */ - def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) + @scala.annotation.varargs + def struct(colName: String, colNames: String*): Column = { + struct((colName +: colNames).map(col): _*) + } + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Map Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Returns a count of the number of times that the regular expression pattern `regexp` is - * matched in the string `str`. + * Creates a new map column. The input columns must be grouped as key-value pairs, e.g. (key1, + * value1, key2, value2, ...). The key columns must all have the same data type, and can't be + * null. The value columns must all have the same data type. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @param cols + * The columns grouped as key-value pairs (key1, value1, key2, value2, ...). Each is a column + * of any type; key columns must share a type and value columns must share a type. + * @group map_funcs + * @since 2.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a map. */ - def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) + @scala.annotation.varargs + def map(cols: Column*): Column = Column.fn("map", cols: _*) /** - * Extract a specific group matched by a Java regex, from the specified string column. If the - * regex did not match, or the specified group did not match, an empty string is returned. if - * the specified group index exceeds the group count of regex, an IllegalArgumentException will - * be thrown. + * Creates a new map column. The array in the first column is used for keys. The array in the + * second column is used for values. All elements in the array for key should not be null. * - * @param e - * target column to work on. A column that evaluates to a string. - * @param exp - * regex pattern to apply. A string. Must be a constant. - * @param groupIdx - * matched group id. An integer. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param keys + * The array of keys for the map; elements must not be null. A column that evaluates to an + * array. + * @param values + * The array of values for the map. A column that evaluates to an array. + * @group map_funcs + * @since 2.4 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = - Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) + def map_from_arrays(keys: Column, values: Column): Column = + Column.fn("map_from_arrays", keys, values) /** - * Extract all strings in the `str` that match the `regexp` expression and corresponding to the - * first regex group index. + * Creates a map after splitting the text into key/value pairs using delimiters. Both + * `pairDelim` and `keyValueDelim` are treated as regular expressions. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. + * @param keyValueDelim + * Delimiter used to split key and value, treated as a regular expression. A column that + * evaluates to a string. + * @group map_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a map. */ - def regexp_extract_all(str: Column, regexp: Column): Column = - Column.fn("regexp_extract_all", str, regexp) + def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = + Column.fn("str_to_map", text, pairDelim, keyValueDelim) /** - * Extract all strings in the `str` that match the `regexp` expression and corresponding to the - * regex group index. + * Creates a map after splitting the text into key/value pairs using delimiters. The `pairDelim` + * is treated as regular expressions. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @param idx - * matched group id. A column that evaluates to an integer. - * @group string_funcs + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. + * @group map_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a map. */ - def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = - Column.fn("regexp_extract_all", str, regexp, idx) + def str_to_map(text: Column, pairDelim: Column): Column = + Column.fn("str_to_map", text, pairDelim) /** - * Replace all substrings of the specified string value that match regexp with rep. + * Creates a map after splitting the text into key/value pairs using delimiters. * - * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A string. Must be a constant. - * @param replacement - * replacement string. A string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @group map_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - def regexp_replace(e: Column, pattern: String, replacement: String): Column = - regexp_replace(e, lit(pattern), lit(replacement)) + def str_to_map(text: Column): Column = Column.fn("str_to_map", text) /** - * Replace all substrings of the specified string value that match regexp with rep, starting at - * the specified position `pos`. - * - * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A string. Must be a constant. - * @param replacement - * replacement string. A string. Must be a constant. - * @param pos - * position to start replacement. The first position is 1. An integer. Must be a constant. - * @group string_funcs - * @since 4.3.0 + * Returns true if the map contains the key. + * @param column + * the input column. A column that evaluates to a map. + * @param key + * the key to check for. A column that evaluates to the map's key type. Must be a constant. + * @group map_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a boolean. */ - def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = - regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) + def map_contains_key(column: Column, key: Any): Column = + Column.fn("map_contains_key", column, lit(key)) /** - * Replace all substrings of the specified string value that match regexp with rep. - * + * Returns an unordered array containing the keys of the map. * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A column that evaluates to a string. - * @param replacement - * replacement string. A column that evaluates to a string. - * @group string_funcs - * @since 2.1.0 + * the input column. A column that evaluates to a map. + * @group map_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = - Column.fn("regexp_replace", e, pattern, replacement) + def map_keys(e: Column): Column = Column.fn("map_keys", e) /** - * Replace all substrings of the specified string value that match regexp with rep, starting at - * the specified position `pos`. - * + * Returns an unordered array containing the values of the map. * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A column that evaluates to a string. - * @param replacement - * replacement string. A column that evaluates to a string. - * @param pos - * position to start replacement. The first position is 1. A column that evaluates to an - * integer. - * @group string_funcs - * @since 4.3.0 + * the input column. A column that evaluates to a map. + * @group map_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = - Column.fn("regexp_replace", e, pattern, replacement, pos) + def map_values(e: Column): Column = Column.fn("map_values", e) /** - * Returns the substring that matches the regular expression `regexp` within the string `str`. - * If the regular expression is not found, the result is null. - * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * Returns an unordered array of all entries in the given map. + * @param e + * the input column. A column that evaluates to a map. + * @group map_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) + def map_entries(e: Column): Column = Column.fn("map_entries", e) /** - * Searches a string for a regular expression and returns an integer that indicates the - * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match - * is found, returns 0. - * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * Returns a map created from the given array of entries. + * @param e + * the array of entries to convert. A column that evaluates to an array of structs, each with + * a key and value field. + * @group map_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a map. */ - def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) + def map_from_entries(e: Column): Column = Column.fn("map_from_entries", e) /** - * Searches a string for a regular expression and returns an integer that indicates the - * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match - * is found, returns 0. - * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @param idx - * matched group id. A column that evaluates to an integer. - * @group string_funcs - * @since 3.5.0 + * Returns the union of all the given maps. + * @param cols + * the maps to merge. Each is a column that evaluates to a map. + * @group map_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a map. */ - def regexp_instr(str: Column, regexp: Column, idx: Column): Column = - Column.fn("regexp_instr", str, regexp, idx) + @scala.annotation.varargs + def map_concat(cols: Column*): Column = Column.fn("map_concat", cols: _*) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Aggregate Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse - * of base64. - * - * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def unbase64(e: Column): Column = Column.fn("unbase64", e) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(e: Column): Column = approx_count_distinct(e) /** - * Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary column. This is - * the reverse of to_base32. - * - * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 4.3.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def from_base32(e: Column): Column = Column.fn("from_base32", e) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) /** - * Right-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. - * - * @param str - * target column to work on. A column that evaluates to a string. - * @param len - * length of the final string. An integer. Must be a constant. - * @param pad - * chars to append. A string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) /** - * Right-pad the binary column with pad to a byte length of len. If the binary column is longer - * than len, the return value is shortened to len bytes. - * - * @param str - * target column to work on. A column that evaluates to a binary. - * @param len - * byte length of the final binary. An integer. Must be a constant. - * @param pad - * bytes to append. A binary. Must be a constant. - * @group string_funcs - * @since 3.3.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(columnName: String, rsd: Double): Column = { + approx_count_distinct(Column(columnName), rsd) + } /** - * Right-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param str - * target column to work on. A column that evaluates to a string or binary. - * @param len - * length of the final result. A column that evaluates to an integer. - * @param pad - * chars or bytes to append. A column that evaluates to a string or binary. - * @group string_funcs - * @since 4.0.0 + * @param e + * The column to count distinct values in. A column of any type. + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) + def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) /** - * Repeats a string column n times, and returns it as a new string column. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param n - * number of times to repeat value. A column that evaluates to an integral. Must be a - * constant. - * @group string_funcs - * @since 1.5.0 + * @param columnName + * The name of the column to count distinct values in. A column of any type. + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) + def approx_count_distinct(columnName: String): Column = approx_count_distinct( + column(columnName)) /** - * Repeats a string column n times, and returns it as a new string column. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param n - * number of times to repeat value. A column that evaluates to an integral. - * @group string_funcs - * @since 4.0.0 + * @param rsd + * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a + * double. Must be a constant. + * + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) + def approx_count_distinct(e: Column, rsd: Double): Column = { + Column.fn("approx_count_distinct", e, lit(rsd)) + } /** - * Trim the spaces from right end for the specified string value. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * @param rsd + * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a + * double. Must be a constant. + * + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def rtrim(e: Column): Column = Column.fn("rtrim", e) + def approx_count_distinct(columnName: String, rsd: Double): Column = { + approx_count_distinct(Column(columnName), rsd) + } /** - * Trim the specified character string from right end for the specified string column. + * Aggregate function: returns the average of the values in a group. + * * @param e - * target column to work on. A column that evaluates to a string. - * @param trimString - * the trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 2.3.0 + * The column to average. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a numeric. */ - def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) + def avg(e: Column): Column = Column.fn("avg", e) /** - * Trim the specified character string from right end for the specified string column. - * @param e - * target column to work on. A column that evaluates to a string. - * @param trim - * the trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * Aggregate function: returns the average of the values in a group. + * + * @param columnName + * The name of the column to average. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a numeric. */ - def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) + def avg(columnName: String): Column = avg(Column(columnName)) /** - * Returns the soundex code for the specified expression. + * Aggregate function: returns a list of objects with duplicates. * * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * The column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def soundex(e: Column): Column = Column.fn("soundex", e) + def collect_list(e: Column): Column = Column.fn("collect_list", e) /** - * Splits str around matches of the given pattern. + * Aggregate function: returns a list of objects with duplicates. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a string representing a regular expression. The regex string should be a Java regular - * expression. A column that evaluates to a string. + * @param columnName + * The name of the column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. * - * @group string_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to an array. */ - def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) + def collect_list(columnName: String): Column = collect_list(Column(columnName)) /** - * Splits str around matches of the given pattern. + * Aggregate function: returns a set of objects with duplicate elements eliminated. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a column of string representing a regular expression. The regex string should be a Java - * regular expression. A column that evaluates to a string. + * @param e + * The column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. * - * @group string_funcs - * @since 4.0.0 + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to an array. */ - def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) + def collect_set(e: Column): Column = Column.fn("collect_set", e) /** - * Splits str around matches of the given pattern. + * Aggregate function: returns a set of objects with duplicate elements eliminated. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a string representing a regular expression. The regex string should be a Java regular - * expression. A column that evaluates to a string. - * @param limit - * an integer expression which controls the number of times the regex is applied.
    - *
  • limit greater than 0: The resulting array's length will not be more than limit, and the - * resulting array's last entry will contain all input beyond the last matched regex.
    • - *
  • limit less than or equal to 0: `regex` will be applied as many times as possible, and - * the resulting array can be of any size.
A column that evaluates to an integer. + * @param columnName + * The name of the column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. * - * @group string_funcs - * @since 3.0.0 + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to an array. */ - def split(str: Column, pattern: String, limit: Int): Column = - Column.fn("split", str, lit(pattern), lit(limit)) + def collect_set(columnName: String): Column = collect_set(Column(columnName)) /** - * Splits str around matches of the given pattern. + * Aggregate function: returns the distinct union of the elements of an array-typed column + * across rows. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a column of string representing a regular expression. The regex string should be a Java - * regular expression. A column that evaluates to a string. - * @param limit - * a column of integer expression which controls the number of times the regex is applied. - *
  • limit greater than 0: The resulting array's length will not be more than limit, - * and the resulting array's last entry will contain all input beyond the last matched - * regex.
  • limit less than or equal to 0: `regex` will be applied as many times as - * possible, and the resulting array can be of any size.
A column that evaluates to - * an integer. + * The aggregation buffer holds only the distinct elements, so its size is bounded by the + * element universe rather than by the number of input rows. Null elements are dropped by + * default (IGNORE NULLS), matching `collect_set`. With `RESPECT NULLS`, a single null element + * is kept, in which case this is equivalent to `array_distinct(flatten(collect_list(e)))`. The + * `RESPECT NULLS` clause is only available through SQL (e.g. + * `expr("collect_union(col) RESPECT NULLS")`). * - * @group string_funcs - * @since 4.0.0 + * @param e + * The array column to collect the union of. A column of type array. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 4.3.0 * @return * Returns a column that evaluates to an array. */ - def split(str: Column, pattern: Column, limit: Column): Column = - Column.fn("split", str, pattern, limit) + def collect_union(e: Column): Column = Column.fn("collect_union", e) /** - * Substring starts at `pos` and is of length `len` when str is String type or returns the slice - * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type + * Aggregate function: returns the distinct union of the elements of an array-typed column + * across rows. * - * @param str - * target column to work on. A column that evaluates to a string or binary. - * @param pos - * starting position in str. A column that evaluates to an integral. Must be a constant. - * @param len - * length of chars. A column that evaluates to an integral. Must be a constant. + * @param columnName + * The name of the array column to collect the union of. A column of type array. * @note - * The position is not zero based, but 1 based index. + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. * - * @group string_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 4.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an array. */ - def substring(str: Column, pos: Int, len: Int): Column = - Column.fn("substring", str, lit(pos), lit(len)) + def collect_union(columnName: String): Column = collect_union(Column(columnName)) /** - * Substring starts at `pos` and is of length `len` when str is String type or returns the slice - * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type - * - * @param str - * target column to work on. A column that evaluates to a string or binary. - * @param pos - * starting position in str. A column that evaluates to an integral. - * @param len - * length of chars. A column that evaluates to an integral. - * @note - * The position is not zero based, but 1 based index. + * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is + * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min + * sketch is a probabilistic data structure used for cardinality estimation using sub-linear + * space. * - * @group string_funcs - * @since 4.0.0 + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. + * @param seed + * The random seed. A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def substring(str: Column, pos: Column, len: Column): Column = - Column.fn("substring", str, pos, len) + def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = + Column.fn("count_min_sketch", e, eps, confidence, seed) /** - * Returns the substring from string str before count occurrences of the delimiter delim. If - * count is positive, everything the left of the final delimiter (counting from left) is - * returned. If count is negative, every to the right of the final delimiter (counting from the - * right) is returned. substring_index performs a case-sensitive match when searching for delim. + * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is + * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min + * sketch is a probabilistic data structure used for cardinality estimation using sub-linear + * space. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param delim - * delimiter of values. A column that evaluates to a string. Must be a constant. - * @param count - * number of occurrences. A column that evaluates to an integral. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def substring_index(str: Column, delim: String, count: Int): Column = - Column.fn("substring_index", str, lit(delim), lit(count)) + def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = + count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) /** - * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of - * `src` and proceeding for `len` bytes. + * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * - * @param src - * the string that will be replaced. A column that evaluates to a string or binary. - * @param replace - * the substitution string. A column that evaluates to a string or binary. - * @param pos - * the starting position in src. A column that evaluates to an integral. - * @param len - * the number of bytes to replace in src. A column that evaluates to an integral. - * @group string_funcs - * @since 3.0.0 + * @param column1 + * The first column. A column that evaluates to a numeric. + * @param column2 + * The second column. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = - Column.fn("overlay", src, replace, pos, len) + def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) /** - * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of - * `src`. + * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * - * @param src - * the string that will be replaced. A column that evaluates to a string or binary. - * @param replace - * the substitution string. A column that evaluates to a string or binary. - * @param pos - * the starting position in src. A column that evaluates to an integral. - * @group string_funcs - * @since 3.0.0 + * @param columnName1 + * The name of the first column. A column that evaluates to a numeric. + * @param columnName2 + * The name of the second column. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def overlay(src: Column, replace: Column, pos: Column): Column = - Column.fn("overlay", src, replace, pos) + def corr(columnName1: String, columnName2: String): Column = { + corr(Column(columnName1), Column(columnName2)) + } /** - * Splits a string into arrays of sentences, where each sentence is an array of words. - * @param string - * a string to be split. A column that evaluates to a string. - * @param language - * a language of the locale. A column that evaluates to a string. - * @param country - * a country of the locale. A column that evaluates to a string. - * @group string_funcs - * @since 3.2.0 + * Aggregate function: returns the number of items in a group. + * + * @param e + * The column to count. A column of any type. + * @group agg_funcs + * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. + */ + def count(e: Column): Column = + Column.fn("count", e) + + /** + * Aggregate function: returns the number of items in a group. + * + * @param columnName + * The name of the column to count. A column of any type. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a long. */ - def sentences(string: Column, language: Column, country: Column): Column = - Column.fn("sentences", string, language, country) + def count(columnName: String): TypedColumn[Any, Long] = + count(Column(columnName)).as(PrimitiveLongEncoder) /** - * Splits a string into arrays of sentences, where each sentence is an array of words. The - * default `country`('') is used. - * @param string - * a string to be split. A column that evaluates to a string. - * @param language - * a language of the locale. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * Aggregate function: returns the number of distinct items in a group. + * + * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. + * + * @param expr + * The first column. A column of any type. + * @param exprs + * Additional columns. A column of any type. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a long. */ - def sentences(string: Column, language: Column): Column = - Column.fn("sentences", string, language) + @scala.annotation.varargs + def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) /** - * Splits a string into arrays of sentences, where each sentence is an array of words. The - * default locale is used. - * @param string - * a string to be split. A column that evaluates to a string. - * @group string_funcs - * @since 3.2.0 + * Aggregate function: returns the number of distinct items in a group. + * + * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. + * + * @param columnName + * first column to compute on. A column of any type. + * @param columnNames + * additional columns to compute on. Columns of any type. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a long. */ - def sentences(string: Column): Column = Column.fn("sentences", string) + @scala.annotation.varargs + def countDistinct(columnName: String, columnNames: String*): Column = + count_distinct(Column(columnName), columnNames.map(Column.apply): _*) /** - * Translate any character in the src by a character in replaceString. The characters in - * replaceString correspond to the characters in matchingString. The translate will happen when - * any character in the string matches the character in the `matchingString`. + * Aggregate function: returns the number of distinct items in a group. * - * @param src - * source column to work on. A column that evaluates to a string. - * @param matchingString - * matching characters. A column that evaluates to a string. Must be a constant. - * @param replaceString - * characters for replacement. A column that evaluates to a string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param expr + * first column to compute on. A column of any type. + * @param exprs + * additional columns to compute on. Columns of any type. + * @group agg_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def translate(src: Column, matchingString: String, replaceString: String): Column = - Column.fn("translate", src, lit(matchingString), lit(replaceString)) + @scala.annotation.varargs + def count_distinct(expr: Column, exprs: Column*): Column = + Column.fn("count", isDistinct = true, expr +: exprs: _*) /** - * Trim the spaces from both ends for the specified string column. + * Aggregate function: returns the population covariance for two columns. * - * @param e - * The string column to trim. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def trim(e: Column): Column = Column.fn("trim", e) + def covar_pop(column1: Column, column2: Column): Column = + Column.fn("covar_pop", column1, column2) /** - * Trim the specified character from both ends for the specified string column. - * @param e - * The string column to trim. A column that evaluates to a string. - * @param trimString - * The trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 2.3.0 + * Aggregate function: returns the population covariance for two columns. + * + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) + def covar_pop(columnName1: String, columnName2: String): Column = { + covar_pop(Column(columnName1), Column(columnName2)) + } /** - * Trim the specified character from both ends for the specified string column. - * @param e - * The string column to trim. A column that evaluates to a string. - * @param trim - * The trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * Aggregate function: returns the sample covariance for two columns. + * + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) + def covar_samp(column1: Column, column2: Column): Column = + Column.fn("covar_samp", column1, column2) /** - * Converts a string column to upper case. + * Aggregate function: returns the sample covariance for two columns. * - * @param e - * The input column to convert to upper case. A column that evaluates to a string. - * @group string_funcs - * @since 1.3.0 + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def upper(e: Column): Column = Column.fn("upper", e) + def covar_samp(columnName1: String, columnName2: String): Column = { + covar_samp(Column(columnName1), Column(columnName2)) + } /** - * Converts the input `e` to a binary value based on the supplied `format`. The `format` can be - * a case-insensitive string literal of "hex", "utf-8", "utf8", or "base64". By default, the - * binary format for conversion is "hex" if `format` is omitted. The function returns NULL if at - * least one of the input parameters is NULL. + * Aggregate function: returns the first value in a group. + * + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * * @param e - * The input value to convert. A column that evaluates to a string. - * @param f - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant. - * @group string_funcs - * @since 3.5.0 + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) + def first(e: Column, ignoreNulls: Boolean): Column = + Column.fn("first", false, e, lit(ignoreNulls)) /** - * Converts the input `e` to a binary value based on the default format "hex". The function - * returns NULL if at least one of the input parameters is NULL. + * Aggregate function: returns the first value of a column in a group. * - * @param e - * The input value to convert. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param columnName + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def to_binary(e: Column): Column = Column.fn("to_binary", e) + def first(columnName: String, ignoreNulls: Boolean): Column = { + first(Column(columnName), ignoreNulls) + } - // scalastyle:off line.size.limit /** - * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. - * The format can consist of the following characters, case insensitive: '0' or '9': Specifies - * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a - * sequence of digits in the input value, generating a result string of the same length as the - * corresponding sequence in the format string. The result string is left-padded with zeros if - * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts - * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': - * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': - * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to - * the left and right of each grouping separator. '$': Specifies the location of the $ currency - * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' - * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note - * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the - * end of the format string; specifies that the result string will be wrapped by angle brackets - * if the input value is negative. + * Aggregate function: returns the first value in a group. * - * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime - * Patterns. If `e` is a binary, it is converted to a string in one of the formats: - * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input - * binary is decoded to UTF-8 string. + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * * @param e - * The input value to convert. A column that evaluates to a numeric, date, timestamp or - * binary. - * @param format - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant when `e` is a numeric or binary value. - * @group string_funcs - * @since 3.5.0 + * column to fetch the first value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - // scalastyle:on line.size.limit - def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) + def first(e: Column): Column = first(e, ignoreNulls = false) - // scalastyle:off line.size.limit /** - * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. - * The format can consist of the following characters, case insensitive: '0' or '9': Specifies - * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a - * sequence of digits in the input value, generating a result string of the same length as the - * corresponding sequence in the format string. The result string is left-padded with zeros if - * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts - * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': - * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': - * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to - * the left and right of each grouping separator. '$': Specifies the location of the $ currency - * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' - * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note - * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the - * end of the format string; specifies that the result string will be wrapped by angle brackets - * if the input value is negative. + * Aggregate function: returns the first value of a column in a group. * - * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime - * Patterns. If `e` is a binary, it is converted to a string in one of the formats: - * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input - * binary is decoded to UTF-8 string. + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * - * @param e - * The input value to convert. A column that evaluates to a numeric, date, timestamp or - * binary. - * @param format - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant when `e` is a numeric or binary value. - * @group string_funcs - * @since 3.5.0 + * @param columnName + * column to fetch the first value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - // scalastyle:on line.size.limit - def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) + def first(columnName: String): Column = first(Column(columnName)) /** - * Convert string 'e' to a number based on the string format 'format'. Throws an exception if - * the conversion fails. The format can consist of the following characters, case insensitive: - * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format - * string matches a sequence of digits in the input string. If the 0/9 sequence starts with 0 - * and is before the decimal point, it can only match a digit sequence of the same size. - * Otherwise, if the sequence starts with 9 or is after the decimal point, it can match a digit - * sequence that has the same or smaller size. '.' or 'D': Specifies the position of the decimal - * point (optional, only allowed once). ',' or 'G': Specifies the position of the grouping - * (thousands) separator (,). There must be a 0 or 9 to the left and right of each grouping - * separator. 'expr' must match the grouping separator relevant for the size of the number. '$': - * Specifies the location of the $ currency sign. This character may only be specified once. 'S' - * or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at the - * beginning or end of the format string). Note that 'S' allows '-' but 'MI' does not. 'PR': - * Only allowed at the end of the format string; specifies that 'expr' indicates a negative - * number with wrapping angled brackets. + * Aggregate function: returns the first value in a group. * * @param e - * The input string to convert to a number. A column that evaluates to a string. - * @param format - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant. - * @group string_funcs + * column to fetch the first value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column of the same type as the input. */ - def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) + def first_value(e: Column): Column = Column.fn("first_value", e) /** - * Replaces all occurrences of `search` with `replace`. + * Aggregate function: returns the first value in a group. * - * @param src - * A column of strings to be replaced. A column that evaluates to a string. - * @param search - * A column of strings. If `search` is not found in `str`, `str` is returned unchanged. A - * column that evaluates to a string. - * @param replace - * A column of strings. If `replace` is not specified or is an empty string, nothing replaces - * the string that is removed from `str`. A column that evaluates to a string. + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * - * @group string_funcs + * @param e + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def replace(src: Column, search: Column, replace: Column): Column = - Column.fn("replace", src, search, replace) + def first_value(e: Column, ignoreNulls: Column): Column = + Column.fn("first_value", e, ignoreNulls) /** - * Replaces all occurrences of `search` with `replace`. - * - * @param src - * A column of strings to be replaced. A column that evaluates to a string. - * @param search - * A column of strings. If `search` is not found in `src`, `src` is returned unchanged. A - * column that evaluates to a string. + * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or + * not, returns 1 for aggregated or 0 for not aggregated in the result set. * - * @group string_funcs - * @since 3.5.0 + * @param e + * column to check if it is aggregated. A column of any type. + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a byte. */ - def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) + def grouping(e: Column): Column = Column.fn("grouping", e) /** - * Splits `str` by delimiter and return requested part of the split (1-based). If any input is - * null, returns null. if `partNum` is out of range of split parts, returns empty string. If - * `partNum` is 0, throws an error. If `partNum` is negative, the parts are counted backward - * from the end of the string. If the `delimiter` is an empty string, the `str` is not split. + * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or + * not, returns 1 for aggregated or 0 for not aggregated in the result set. * - * @param str - * A column of strings to be split. A column that evaluates to a string. - * @param delimiter - * The delimiter used for split. A column that evaluates to a string. - * @param partNum - * The requested part of the split (1-based). A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * @param columnName + * column to check if it is aggregated. A column of any type. + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a byte. */ - def split_part(str: Column, delimiter: Column, partNum: Column): Column = - Column.fn("split_part", str, delimiter, partNum) + def grouping(columnName: String): Column = grouping(Column(columnName)) /** - * Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of - * byte array that starts at `pos` and is of length `len`. + * Aggregate function: returns the level of grouping, equals to * - * @param str - * The input from which to take the substring. A column that evaluates to a string or binary. - * @param pos - * The starting position of the substring. A column that evaluates to an integral. - * @param len - * The length of the substring. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * {{{ + * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) + * }}} + * + * @param cols + * columns to check for. Columns of any type. + * @note + * The list of columns should match with grouping columns exactly, or empty (means all the + * grouping columns). + * + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def substr(str: Column, pos: Column, len: Column): Column = - Column.fn("substr", str, pos, len) + @scala.annotation.varargs + def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) /** - * Returns the substring of `str` that starts at `pos`, or the slice of byte array that starts - * at `pos`. + * Aggregate function: returns the level of grouping, equals to * - * @param str - * The input from which to take the substring. A column that evaluates to a string or binary. - * @param pos - * The starting position of the substring. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * {{{ + * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) + * }}} + * + * @param colName + * the name of the first grouping column. A column of any type. + * @param colNames + * the names of the remaining grouping columns. Columns of any type. + * @note + * The list of columns should match with grouping columns exactly. + * + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) + @scala.annotation.varargs + def grouping_id(colName: String, colNames: String*): Column = { + grouping_id((Seq(colName) ++ colNames).map(n => Column(n)): _*) + } /** - * Extracts a part from a URL. + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with lgConfigK arg. * - * @param url - * A column of strings, each representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @param key - * The key of a query parameter in the URL. A column that evaluates to a string. - * @group url_funcs - * @since 4.0.0 + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = - Column.fn("try_parse_url", url, partToExtract, key) + def hll_sketch_agg(e: Column, lgConfigK: Column): Column = + Column.fn("hll_sketch_agg", e, lgConfigK) /** - * Extracts a part from a URL. + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with lgConfigK arg. * - * @param url - * A column of strings, each representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @group url_funcs - * @since 4.0.0 + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. + */ + def hll_sketch_agg(e: Column, lgConfigK: Int): Column = + Column.fn("hll_sketch_agg", e, lit(lgConfigK)) + + /** + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with lgConfigK arg. + * + * @param columnName + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def try_parse_url(url: Column, partToExtract: Column): Column = - Column.fn("try_parse_url", url, partToExtract) + def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { + hll_sketch_agg(Column(columnName), lgConfigK) + } /** - * Extracts a part from a URL. + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with default lgConfigK value. * - * @param url - * A column of strings, each representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @param key - * The key of a query parameter in the URL. A column that evaluates to a string. - * @group url_funcs + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def parse_url(url: Column, partToExtract: Column, key: Column): Column = - Column.fn("parse_url", url, partToExtract, key) + def hll_sketch_agg(e: Column): Column = + Column.fn("hll_sketch_agg", e) /** - * Extracts a part from a URL. + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with default lgConfigK value. * - * @param url - * A column representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @group url_funcs + * @param columnName + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def parse_url(url: Column, partToExtract: Column): Column = - Column.fn("parse_url", url, partToExtract) + def hll_sketch_agg(columnName: String): Column = { + hll_sketch_agg(Column(columnName)) + } /** - * Formats the arguments in printf-style and returns the result as a string column. + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values + * and allowDifferentLgConfigK is set to false. * - * @param format - * A format string that can contain embedded format tags. A column that evaluates to a string. - * @param arguments - * The values to be used in formatting. Columns that evaluate to any type. - * @group string_funcs + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - @scala.annotation.varargs - def printf(format: Column, arguments: Column*): Column = - Column.fn("printf", (format +: arguments): _*) + def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = + Column.fn("hll_union_agg", e, allowDifferentLgConfigK) /** - * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding - * scheme. + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values + * and allowDifferentLgConfigK is set to false. * - * @param str - * A URL-encoded string. A column that evaluates to a string. - * @group url_funcs + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def url_decode(str: Column): Column = Column.fn("url_decode", str) + def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = + Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) /** - * This is a special version of `url_decode` that performs the same operation, but returns a - * NULL value instead of raising an error if the decoding cannot be performed. + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values + * and allowDifferentLgConfigK is set to false. * - * @param str - * A URL-encoded string. A column that evaluates to a string. - * @group url_funcs - * @since 4.0.0 + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) + def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { + hll_union_agg(Column(columnName), allowDifferentLgConfigK) + } /** - * Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding - * scheme. + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. * - * @param str - * A string to encode. A column that evaluates to a string. - * @group url_funcs + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def url_encode(str: Column): Column = Column.fn("url_encode", str) + def hll_union_agg(e: Column): Column = + Column.fn("hll_union_agg", e) /** - * Returns the position of the first occurrence of `substr` in `str` after position `start`. The - * given `start` and return value are 1-based. + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. * - * @param substr - * The substring to search for. A column that evaluates to a string. - * @param str - * The string to search in. A column that evaluates to a string. - * @param start - * The 1-based position to start the search from. A column that evaluates to an integral. - * @group string_funcs + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def position(substr: Column, str: Column, start: Column): Column = - Column.fn("position", substr, str, start) + def hll_union_agg(columnName: String): Column = { + hll_union_agg(Column(columnName)) + } /** - * Returns the position of the first occurrence of `substr` in `str` after position `1`. The - * return value are 1-based. + * Aggregate function: returns the kurtosis of the values in a group. * - * @param substr - * The substring to search for. A column that evaluates to a string. - * @param str - * The string to search in. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @param e + * the column to compute the kurtosis on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def position(substr: Column, str: Column): Column = - Column.fn("position", substr, str) + def kurtosis(e: Column): Column = Column.fn("kurtosis", e) /** - * Returns a boolean. The value is True if str ends with suffix. Returns NULL if either input - * expression is NULL. Otherwise, returns False. Both str or suffix must be of STRING or BINARY - * type. + * Aggregate function: returns the kurtosis of the values in a group. * - * @param str - * The string to test. A column that evaluates to a string or binary. - * @param suffix - * The suffix to test for. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @param columnName + * the name of the column to compute the kurtosis on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a double. */ - def endswith(str: Column, suffix: Column): Column = - Column.fn("endswith", str, suffix) + def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) /** - * Returns a boolean. The value is True if str starts with prefix. Returns NULL if either input - * expression is NULL. Otherwise, returns False. Both str or prefix must be of STRING or BINARY - * type. + * Aggregate function: returns the last value in a group. * - * @param str - * The string to test. A column that evaluates to a string or binary. - * @param prefix - * The prefix to test for. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param e + * the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. A + * column that evaluates to a boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column of the same type as the input. */ - def startswith(str: Column, prefix: Column): Column = - Column.fn("startswith", str, prefix) + def last(e: Column, ignoreNulls: Boolean): Column = + Column.fn("last", false, e, lit(ignoreNulls)) /** - * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the - * result is equivalent to char(n % 256) + * Aggregate function: returns the last value of the column in a group. * - * @param n - * The code point value. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param columnName + * the name of the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. A + * column that evaluates to a boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def char(n: Column): Column = Column.fn("char", n) + def last(columnName: String, ignoreNulls: Boolean): Column = { + last(Column(columnName), ignoreNulls) + } /** - * Removes the leading and trailing space characters from `str`. + * Aggregate function: returns the last value in a group. * - * @param str - * The string to trim. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param e + * column to fetch the last value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def btrim(str: Column): Column = Column.fn("btrim", str) + def last(e: Column): Column = last(e, ignoreNulls = false) /** - * Remove the leading and trailing `trim` characters from `str`. + * Aggregate function: returns the last value of the column in a group. * - * @param str - * The string to trim. A column that evaluates to a string. - * @param trim - * The trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) + def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) /** - * This is a special version of `to_binary` that performs the same operation, but returns a NULL - * value instead of raising an error if the conversion cannot be performed. + * Aggregate function: returns the last value in a group. * * @param e - * The string to convert. A column that evaluates to a string. - * @param f - * The format to use for the conversion. A column that evaluates to a string. Must be a - * constant. - * @group string_funcs + * column to fetch the last value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) + def last_value(e: Column): Column = Column.fn("last_value", e) /** - * This is a special version of `to_binary` that performs the same operation, but returns a NULL - * value instead of raising an error if the conversion cannot be performed. + * Aggregate function: returns the last value in a group. + * + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * * @param e - * The string to convert. A column that evaluates to a string. - * @group string_funcs + * column to fetch the last value for. A column of any type. + * @param ignoreNulls + * whether to skip null values. A column that evaluates to a boolean. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) + def last_value(e: Column, ignoreNulls: Column): Column = + Column.fn("last_value", e, ignoreNulls) /** - * Convert string `e` to a number based on the string format `format`. Returns NULL if the - * string `e` does not match the expected format. The format follows the same semantics as the - * to_number function. + * Aggregate function: returns the most frequent value in a group. * * @param e - * The string to convert. A column that evaluates to a string. - * @param format - * The format used to convert the string to a number. A column that evaluates to a string. - * Must be a constant. - * @group string_funcs - * @since 3.5.0 + * target column to compute on. A column of any type. + * @group agg_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column of the same type as the input. */ - def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) + def mode(e: Column): Column = Column.fn("mode", e) /** - * Returns the character length of string data or number of bytes of binary data. The length of - * string data includes the trailing spaces. The length of binary data includes binary zeros. + * Aggregate function: returns the most frequent value in a group. * - * @param str - * Input column or strings. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to an integer. - */ - def char_length(str: Column): Column = Column.fn("char_length", str) - - /** - * Returns the character length of string data or number of bytes of binary data. The length of - * string data includes the trailing spaces. The length of binary data includes binary zeros. + * When multiple values have the same greatest frequency then either any of values is returned + * if deterministic is false or is not defined, or the lowest value is returned if deterministic + * is true. * - * @param str - * Input column or strings. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @param e + * target column to compute on. A column of any type. + * @param deterministic + * if there are multiple equally-frequent results then return the lowest. A boolean. Must be a + * constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def character_length(str: Column): Column = Column.fn("character_length", str) + def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) /** - * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the - * result is equivalent to chr(n % 256) + * Aggregate function: returns the maximum value of the expression in a group. * - * @param n - * The code point. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * @param e + * the target column on which the maximum value is computed. A column of any type. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def chr(n: Column): Column = Column.fn("chr", n) + def max(e: Column): Column = Column.fn("max", e) /** - * Returns a boolean. The value is True if right is found inside left. Returns NULL if either - * input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or - * BINARY type. + * Aggregate function: returns the maximum value of the column in a group. * - * @param left - * The input to check, may be NULL. A column that evaluates to a string or binary. - * @param right - * The input to find, may be NULL. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column of the same type as the input. */ - def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) + def max(columnName: String): Column = max(Column(columnName)) /** - * Returns the `n`-th input, e.g., returns `input2` when `n` is 2. The function returns NULL if - * the index exceeds the length of the array and `spark.sql.ansi.enabled` is set to false. If - * `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException for invalid - * indices. + * Aggregate function: returns the value associated with the maximum value of ord. * - * @param inputs - * The index followed by the inputs to select from. Columns where the first evaluates to an - * integral and the rest evaluate to strings or binaries. - * @group string_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a string. - */ - @scala.annotation.varargs - def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) - - /** - * Returns the index (1-based) of the given string (`str`) in the comma-delimited list - * (`strArray`). Returns 0, if the string was not found or if the given string (`str`) contains - * a comma. + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @note + * The function is non-deterministic so the output order can be different for those associated + * the same values of `e`. * - * @param str - * The given string to be found. A column that evaluates to a string. - * @param strArray - * The comma-delimited list. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @group agg_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) + def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) /** - * Returns true if str matches `pattern` with `escapeChar`, null if any arguments are null, - * false otherwise. + * Aggregate function: returns an array of values associated with the top `k` values of `ord`. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @param escapeChar - * The escape character. A column that evaluates to a string. Must be a constant. - * @group predicate_funcs - * @since 3.5.0 + * The result array contains values in descending order by their associated ordering values. + * Returns null if there are no non-null ordering values. + * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. An integer. Must be a constant. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. + * + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def like(str: Column, pattern: Column, escapeChar: Column): Column = - Column.fn("like", str, pattern, escapeChar) + def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) /** - * Returns true if str matches `pattern` with `escapeChar`('\'), null if any arguments are null, - * false otherwise. + * Aggregate function: returns an array of values associated with the top `k` values of `ord`. + * + * The result array contains values in descending order by their associated ordering values. + * Returns null if there are no non-null ordering values. + * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. A column that evaluates to an integer. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) + def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) /** - * Returns true if str matches `pattern` with `escapeChar` case-insensitively, null if any - * arguments are null, false otherwise. + * Aggregate function: returns the average of the values in a group. Alias for avg. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @param escapeChar - * The escape character. A column that evaluates to a string. Must be a constant. - * @group predicate_funcs - * @since 3.5.0 + * @param e + * target column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a double. */ - def ilike(str: Column, pattern: Column, escapeChar: Column): Column = - Column.fn("ilike", str, pattern, escapeChar) + def mean(e: Column): Column = avg(e) /** - * Returns true if str matches `pattern` with `escapeChar`('\') case-insensitively, null if any - * arguments are null, false otherwise. + * Aggregate function: returns the average of the values in a group. Alias for avg. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * @group agg_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a double. */ - def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) + def mean(columnName: String): Column = avg(columnName) /** - * Returns `str` with all characters changed to lowercase. + * Aggregate function: returns the median of the values in a group. * - * @param str - * A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @param e + * target column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def lcase(str: Column): Column = Column.fn("lcase", str) + def median(e: Column): Column = Column.fn("median", e) /** - * Returns `str` with all characters changed to uppercase. + * Aggregate function: returns the minimum value of the expression in a group. * - * @param str - * A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @param e + * the target column on which the minimum value is computed. A column of any type. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def ucase(str: Column): Column = Column.fn("ucase", str) + def min(e: Column): Column = Column.fn("min", e) /** - * Returns the leftmost `len`(`len` can be string type) characters from the string `str`, if - * `len` is less or equal than 0 the result is an empty string. + * Aggregate function: returns the minimum value of the column in a group. * - * @param str - * Input column or strings. A column that evaluates to a string or binary. - * @param len - * The number of leftmost characters. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * @param columnName + * the name of the column on which the minimum value is computed. A column of an orderable + * type. + * @group agg_funcs + * @since 1.3.0 * @return * Returns a column of the same type as the input. */ - def left(str: Column, len: Column): Column = Column.fn("left", str, len) + def min(columnName: String): Column = min(Column(columnName)) /** - * Returns the rightmost `len`(`len` can be string type) characters from the string `str`, if - * `len` is less or equal than 0 the result is an empty string. + * Aggregate function: returns the value associated with the minimum value of ord. * - * @param str - * Input column or strings. A column that evaluates to a string. - * @param len - * The number of rightmost characters. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @note + * The function is non-deterministic so the output order can be different for those associated + * the same values of `e`. + * + * @group agg_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def right(str: Column, len: Column): Column = Column.fn("right", str, len) + def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) /** - * Returns `str` enclosed by single quotes and each instance of single quote in it is preceded - * by a backslash. + * Aggregate function: returns an array of values associated with the bottom `k` values of + * `ord`. * - * @param str - * A column that evaluates to a string. - * @group string_funcs - * @since 4.1.0 + * The result array contains values in ascending order by their associated ordering values. + * Returns null if there are no non-null ordering values. + * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. An integer. Must be a constant. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. + * + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def quote(str: Column): Column = Column.fn("quote", str) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Datasketch functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches HllSketch. + * Aggregate function: returns an array of values associated with the bottom `k` values of + * `ord`. * - * @param c - * The binary representation of a Datasketches HllSketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 3.5.0 + * The result array contains values in ascending order by their associated ordering values. + * Returns null if there are no non-null ordering values. + * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. A column that evaluates to an integral. Must be a + * constant. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. + * + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an array. */ - def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) + def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches HllSketch. + * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given + * percentage(s) with value range in [0.0, 1.0]. * - * @param columnName - * Name of the column containing the binary representation of a Datasketches HllSketch. A - * column that evaluates to a binary. - * @group sketch_funcs + * @param e + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a double. */ - def hll_sketch_estimate(columnName: String): Column = { - hll_sketch_estimate(Column(columnName)) - } + def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values. + * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given + * percentage(s) with value range in [0.0, 1.0]. * - * @param c1 - * The first binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @group sketch_funcs + * @param e + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param frequency + * the positive frequency with which to weight each value. A column that evaluates to an + * integral. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def hll_union(c1: Column, c2: Column): Column = - Column.fn("hll_union", c1, c2) + def percentile(e: Column, percentage: Column, frequency: Column): Column = + Column.fn("percentile", e, percentage, frequency) /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values. + * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is + * the smallest value in the ordered `col` values (sorted from least to greatest) such that no + * more than `percentage` of `col` values is less than the value or equal to that value. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @group sketch_funcs - * @since 3.5.0 + * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating + * point value, it must be between 0.0 and 1.0. + * + * The accuracy parameter is a positive numeric literal which controls approximation accuracy at + * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the + * relative error of the approximation. + * + * @param e + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def hll_union(columnName1: String, columnName2: String): Column = { - hll_union(Column(columnName1), Column(columnName2)) - } + def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = + Column.fn("percentile_approx", e, percentage, accuracy) /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values and - * allowDifferentLgConfigK is set to false. + * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is + * the smallest value in the ordered `col` values (sorted from least to greatest) such that no + * more than `percentage` of `col` values is less than the value or equal to that value. * - * @param c1 - * The first binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @param allowDifferentLgConfigK - * Allow sketches with different lgConfigK values to be merged (defaults to false). A column - * that evaluates to a boolean. Must be a constant. - * @group sketch_funcs + * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating + * point value, it must be between 0.0 and 1.0. + * + * The accuracy parameter is a positive numeric literal which controls approximation accuracy at + * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the + * relative error of the approximation. + * + * @param e + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = - Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) + def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { + Column.fn("approx_percentile", e, percentage, accuracy) + } /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values and - * allowDifferentLgConfigK is set to false. + * Aggregate function: returns the product of all numerical elements in a group. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @param allowDifferentLgConfigK - * Allow sketches with different lgConfigK values to be merged (defaults to false). A column - * that evaluates to a boolean. Must be a constant. - * @group sketch_funcs - * @since 3.5.0 + * @param e + * the column to compute the product on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def hll_union( - columnName1: String, - columnName2: String, - allowDifferentLgConfigK: Boolean): Column = { - hll_union(Column(columnName1), Column(columnName2), allowDifferentLgConfigK) - } + def product(e: Column): Column = Column.internalFn("product", e) /** - * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches AnotB object + * Aggregate function: returns the skewness of the values in a group. * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute the skewness on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_difference(c1: Column, c2: Column): Column = - Column.fn("theta_difference", c1, c2) + def skewness(e: Column): Column = Column.fn("skewness", e) /** - * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches AnotB object + * Aggregate function: returns the skewness of the values in a group. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * the name of the column to compute the skewness on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_difference(columnName1: String, columnName2: String): Column = { - theta_difference(Column(columnName1), Column(columnName2)) - } + def skewness(columnName: String): Column = skewness(Column(columnName)) /** - * Intersects two binary representations of Datasketches ThetaSketch objects in the input - * columns using a Datasketches Intersection object + * Aggregate function: alias for `stddev_samp`. * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_intersection(c1: Column, c2: Column): Column = - Column.fn("theta_intersection", c1, c2) + def std(e: Column): Column = Column.fn("std", e) /** - * Intersects two binary representations of Datasketches ThetaSketch objects in the input - * columns using a Datasketches Intersection object + * Aggregate function: alias for `stddev_samp`. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_intersection(columnName1: String, columnName2: String): Column = { - theta_intersection(Column(columnName1), Column(columnName2)) - } + def stddev(e: Column): Column = Column.fn("stddev", e) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches ThetaSketch. + * Aggregate function: alias for `stddev_samp`. * - * @param c - * The binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * the name of the column to compute the standard deviation on. A column that evaluates to a + * numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a double. */ - def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) + def stddev(columnName: String): Column = stddev(Column(columnName)) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches ThetaSketch. + * Aggregate function: returns the sample standard deviation of the expression in a group. * - * @param columnName - * Name of the column containing the binary representation of a Datasketches ThetaSketch. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute the sample standard deviation on. A column that evaluates to a + * numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a double. */ - def theta_sketch_estimate(columnName: String): Column = { - theta_sketch_estimate(Column(columnName)) - } + def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It is configured with the default value of 12 for - * `lgNomEntries`. + * Aggregate function: returns the sample standard deviation of the expression in a group. * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * Name of the column to compute the sample standard deviation on. A column that evaluates to + * a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union(c1: Column, c2: Column): Column = - Column.fn("theta_union", c1, c2) + def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It is configured with the default value of 12 for - * `lgNomEntries`. + * Aggregate function: returns the population standard deviation of the expression in a group. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * The column to compute the population standard deviation on. A column that evaluates to a + * numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union(columnName1: String, columnName2: String): Column = { - theta_union(Column(columnName1), Column(columnName2)) - } + def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Aggregate function: returns the population standard deviation of the expression in a group. * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * Name of the column to compute the population standard deviation on. A column that evaluates + * to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("theta_union", c1, c2, lit(lgNomEntries)) + def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Aggregate function: returns the sum of all values in the expression. * - * @param columnName1 - * The first ThetaSketch column to union. A column that evaluates to a binary. - * @param columnName2 - * The second ThetaSketch column to union. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * The column to sum. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a numeric or interval. */ - def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { - theta_union(Column(columnName1), Column(columnName2), lgNomEntries) - } + def sum(e: Column): Column = Column.fn("sum", e) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Aggregate function: returns the sum of all values in the given column. * - * @param c1 - * The first ThetaSketch column to union. A column that evaluates to a binary. - * @param c2 - * The second ThetaSketch column to union. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * Name of the column to sum. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a numeric or interval. */ - def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = - Column.fn("theta_union", c1, c2, lgNomEntries) + def sum(columnName: String): Column = sum(Column(columnName)) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. + * Aggregate function: returns the sum of distinct values in the expression. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a numeric or interval. */ - def tuple_difference_double(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_double", c1, c2) + @deprecated("Use sum_distinct", "3.2.0") + def sumDistinct(e: Column): Column = sum_distinct(e) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. + * Aggregate function: returns the sum of distinct values in the expression. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a numeric or interval. */ - def tuple_difference_double(columnName1: String, columnName2: String): Column = - tuple_difference_double(Column(columnName1), Column(columnName2)) + @deprecated("Use sum_distinct", "3.2.0") + def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. + * Aggregate function: returns the sum of distinct values in the expression. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column to sum distinct values of. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a numeric or interval. */ - def tuple_difference_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_integer", c1, c2) + def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input + * column via a Datasketches Intersection instance. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column of Datasketches ThetaSketch instances to intersect. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_difference_integer(columnName1: String, columnName2: String): Column = - tuple_difference_integer(Column(columnName1), Column(columnName2)) + def theta_intersection_agg(e: Column): Column = + Column.fn("theta_intersection_agg", e) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input + * volumn via a Datasketches Intersection instance. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * Name of the column of Datasketches ThetaSketch instances to intersect. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_double(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_double", c1, c2) + def theta_intersection_agg(columnName: String): Column = + theta_intersection_agg(Column(columnName)) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the `lgNomEntries` nominal + * entries. * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_double(columnName1: String, columnName2: String): Column = - tuple_intersection_double(Column(columnName1), Column(columnName2)) + def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = + Column.fn("theta_sketch_agg", e, lgNomEntries) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the `lgNomEntries` nominal + * entries. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_double", c1, c2, lit(mode)) + def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = + Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the `lgNomEntries` nominal + * entries. * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = - tuple_intersection_double(Column(columnName1), Column(columnName2), mode) + def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = + theta_sketch_agg(Column(columnName), lgNomEntries) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the default value of 12 for + * `lgNomEntries`. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_double", c1, c2, mode) + def theta_sketch_agg(e: Column): Column = + Column.fn("theta_sketch_agg", e) /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the default value of 12 for + * `lgNomEntries`. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_integer", c1, c2) + def theta_sketch_agg(columnName: String): Column = + theta_sketch_agg(Column(columnName)) /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_integer(columnName1: String, columnName2: String): Column = - tuple_intersection_integer(Column(columnName1), Column(columnName2)) + def theta_union_agg(e: Column, lgNomEntries: Column): Column = + Column.fn("theta_union_agg", e, lgNomEntries) /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) + def theta_union_agg(e: Column, lgNomEntries: Int): Column = + Column.fn("theta_union_agg", e, lit(lgNomEntries)) /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = - tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) + def theta_union_agg(columnName: String, lgNomEntries: Int): Column = + theta_union_agg(Column(columnName), lgNomEntries) /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It is configured with the default value of 12 for + * `lgNomEntries`. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_integer", c1, c2, mode) + def theta_union_agg(e: Column): Column = + Column.fn("theta_union_agg", e) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with double summary data type. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It is configured with the default value of 12 for + * `lgNomEntries`. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_estimate_double(c: Column): Column = - Column.fn("tuple_sketch_estimate_double", c) + def theta_union_agg(columnName: String): Column = + theta_union_agg(Column(columnName)) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with double summary data type. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. The mode parameter specifies + * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a * binary. - * @group sketch_funcs + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_estimate_double(columnName: String): Column = - tuple_sketch_estimate_double(Column(columnName)) + def tuple_intersection_agg_double(e: Column, mode: Column): Column = + Column.fn("tuple_intersection_agg_double", e, mode) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with integer summary data type. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. The mode parameter specifies + * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a * binary. - * @group sketch_funcs + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_estimate_integer(c: Column): Column = - Column.fn("tuple_sketch_estimate_integer", c) + def tuple_intersection_agg_double(e: Column, mode: String): Column = + Column.fn("tuple_intersection_agg_double", e, lit(mode)) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with integer summary data type. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. The mode parameter specifies + * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_estimate_integer(columnName: String): Column = - tuple_sketch_estimate_integer(Column(columnName)) + def tuple_intersection_agg_double(columnName: String, mode: String): Column = + tuple_intersection_agg_double(Column(columnName), mode) /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. It is configured with the + * default mode of 'sum'. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a * binary. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_double(c: Column): Column = - Column.fn("tuple_sketch_summary_double", c) + def tuple_intersection_agg_double(e: Column): Column = + Column.fn("tuple_intersection_agg_double", e) /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. It is configured with the + * default mode of 'sum'. * * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. - */ - def tuple_sketch_summary_double(columnName: String): Column = - tuple_sketch_summary_double(Column(columnName)) + * Returns a column that evaluates to a binary. + */ + def tuple_intersection_agg_double(columnName: String): Column = + tuple_intersection_agg_double(Column(columnName)) /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a * binary. * @param mode * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to * a string. Must be a constant. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_double(c: Column, mode: String): Column = - Column.fn("tuple_sketch_summary_double", c, lit(mode)) + def tuple_intersection_agg_integer(e: Column, mode: Column): Column = + Column.fn("tuple_intersection_agg_integer", e, mode) /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a * binary. * @param mode * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to * a string. Must be a constant. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_double(columnName: String, mode: String): Column = - tuple_sketch_summary_double(Column(columnName), mode) + def tuple_intersection_agg_integer(e: Column, mode: String): Column = + Column.fn("tuple_intersection_agg_integer", e, lit(mode)) /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @param mode * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to * a string. Must be a constant. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_double(c: Column, mode: Column): Column = - Column.fn("tuple_sketch_summary_double", c, mode) + def tuple_intersection_agg_integer(columnName: String, mode: String): Column = + tuple_intersection_agg_integer(Column(columnName), mode) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. It is configured with + * the default mode of 'sum'. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a * binary. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_integer(c: Column): Column = - Column.fn("tuple_sketch_summary_integer", c) + def tuple_intersection_agg_integer(e: Column): Column = + Column.fn("tuple_intersection_agg_integer", e) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. It is configured with + * the default mode of 'sum'. * * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_integer(columnName: String): Column = - tuple_sketch_summary_integer(Column(columnName)) + def tuple_intersection_agg_integer(columnName: String): Column = + tuple_intersection_agg_integer(Column(columnName)) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_integer(c: Column, mode: String): Column = - Column.fn("tuple_sketch_summary_integer", c, lit(mode)) + def tuple_sketch_agg_double( + key: Column, + summary: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_sketch_agg_double", key, summary, lgNomEntries, mode) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_summary_integer(columnName: String, mode: String): Column = - tuple_sketch_summary_integer(Column(columnName), mode) + def tuple_sketch_agg_double( + key: Column, + summary: Column, + lgNomEntries: Int, + mode: String): Column = + Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries), lit(mode)) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a long. - */ - def tuple_sketch_summary_integer(c: Column, mode: Column): Column = - Column.fn("tuple_sketch_summary_integer", c, mode) - - /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. - * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_theta_double(c: Column): Column = - Column.fn("tuple_sketch_theta_double", c) + def tuple_sketch_agg_double( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int, + mode: String): Column = + tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_theta_double(columnName: String): Column = - tuple_sketch_theta_double(Column(columnName)) + def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = + Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_theta_integer(c: Column): Column = - Column.fn("tuple_sketch_theta_integer", c) + def tuple_sketch_agg_double( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int): Column = + tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @group agg_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def tuple_sketch_theta_integer(columnName: String): Column = - tuple_sketch_theta_integer(Column(columnName)) + def tuple_sketch_agg_double(key: Column, summary: Column): Column = + Column.fn("tuple_sketch_agg_double", key, summary) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_double(c1: Column, c2: Column): Column = - Column.fn("tuple_union_double", c1, c2) + def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = + tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_double(columnName1: String, columnName2: String): Column = - tuple_union_double(Column(columnName1), Column(columnName2)) + def tuple_sketch_agg_integer( + key: Column, + summary: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_sketch_agg_integer", key, summary, lgNomEntries, mode) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an * integral. Must be a constant. - * @group sketch_funcs + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) + def tuple_sketch_agg_integer( + key: Column, + summary: Column, + lgNomEntries: Int, + mode: String): Column = + Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries), lit(mode)) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an * integral. Must be a constant. - * @group sketch_funcs + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = - tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) + def tuple_sketch_agg_integer( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int, + mode: String): Column = + tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an * integral. Must be a constant. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) + def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = + Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an * integral. Must be a constant. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_double( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) + def tuple_sketch_agg_integer( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int): Column = + tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an * integral. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. - * @group sketch_funcs + * @group agg_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. + */ + def tuple_sketch_agg_integer(key: Column, summary: Column): Column = + Column.fn("tuple_sketch_agg_integer", key, summary) + + /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) + def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = + tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param e + * the column containing binary TupleSketch representations to union. A column that evaluates + * to a binary. + * @param lgNomEntries + * the log-base-2 of nominal entries for the union buffer (must be between 4 and 26). A column + * that evaluates to an integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_union_integer", c1, c2) + def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_integer(columnName1: String, columnName2: String): Column = - tuple_union_integer(Column(columnName1), Column(columnName2)) + def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @group sketch_funcs + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) + def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = + tuple_union_agg_double(Column(columnName), lgNomEntries, mode) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @group sketch_funcs + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = - tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) + def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. * @param lgNomEntries - * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a - * constant. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) + def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = + tuple_union_agg_double(Column(columnName), lgNomEntries) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a - * constant. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_integer( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) + def tuple_union_agg_double(e: Column): Column = + Column.fn("tuple_union_agg_double", e) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries. A column that evaluates to an integral. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * @group sketch_funcs + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) + def tuple_union_agg_double(columnName: String): Column = + tuple_union_agg_double(Column(columnName)) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * double summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_difference_theta_double(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_theta_double", c1, c2) + def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * double summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = - tuple_difference_theta_double(Column(columnName1), Column(columnName2)) + def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * integer summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_difference_theta_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_theta_integer", c1, c2) + def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = + tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * integer summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = - tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) + def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_double(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_theta_double", c1, c2) + def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = + tuple_union_agg_integer(Column(columnName), lgNomEntries) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = - tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) + def tuple_union_agg_integer(e: Column): Column = + Column.fn("tuple_union_agg_integer", e) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) + def tuple_union_agg_integer(columnName: String): Column = + tuple_union_agg_integer(Column(columnName)) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_double( - columnName1: String, - columnName2: String, - mode: String): Column = - tuple_intersection_theta_double(Column(columnName1), Column(columnName2), mode) + def kll_sketch_agg_bigint(e: Column, k: Column): Column = + Column.fn("kll_sketch_agg_bigint", e, k) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_theta_double", c1, c2, mode) + def kll_sketch_agg_bigint(e: Column, k: Int): Column = + Column.fn("kll_sketch_agg_bigint", e, lit(k)) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_theta_integer", c1, c2) + def kll_sketch_agg_bigint(columnName: String, k: Int): Column = + kll_sketch_agg_bigint(Column(columnName), k) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column with default k value of 200. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = - tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) + def kll_sketch_agg_bigint(e: Column): Column = + Column.fn("kll_sketch_agg_bigint", e) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column with default k value of 200. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) + def kll_sketch_agg_bigint(columnName: String): Column = + kll_sketch_agg_bigint(Column(columnName)) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). - * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). + * + * @param e + * The column containing float values to aggregate. A column that evaluates to a float. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_integer( - columnName1: String, - columnName2: String, - mode: String): Column = - tuple_intersection_theta_integer(Column(columnName1), Column(columnName2), mode) + def kll_sketch_agg_float(e: Column, k: Column): Column = + Column.fn("kll_sketch_agg_float", e, k) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_theta_integer", c1, c2, mode) + def kll_sketch_agg_float(e: Column, k: Int): Column = + Column.fn("kll_sketch_agg_float", e, lit(k)) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_double(c1: Column, c2: Column): Column = - Column.fn("tuple_union_theta_double", c1, c2) + def kll_sketch_agg_float(columnName: String, k: Int): Column = + kll_sketch_agg_float(Column(columnName), k) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column with default k value of 200. * - * @param columnName1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_double(columnName1: String, columnName2: String): Column = - tuple_union_theta_double(Column(columnName1), Column(columnName2)) + def kll_sketch_agg_float(e: Column): Column = + Column.fn("kll_sketch_agg_float", e) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column with default k value of 200. * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) + def kll_sketch_agg_float(columnName: String): Column = + kll_sketch_agg_float(Column(columnName)) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param columnName1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * @param e + * The column containing double values to aggregate. A column that evaluates to a float or + * double. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_double( - columnName1: String, - columnName2: String, - lgNomEntries: Int): Column = - tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries) + def kll_sketch_agg_double(e: Column, k: Column): Column = + Column.fn("kll_sketch_agg_double", e, k) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * @param e + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) + def kll_sketch_agg_double(e: Column, k: Int): Column = + Column.fn("kll_sketch_agg_double", e, lit(k)) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param columnName1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_double( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) + def kll_sketch_agg_double(columnName: String, k: Int): Column = + kll_sketch_agg_double(Column(columnName), k) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column with default k value of 200. * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_double( - c1: Column, - c2: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_union_theta_double", c1, c2, lgNomEntries, mode) + def kll_sketch_agg_double(e: Column): Column = + Column.fn("kll_sketch_agg_double", e) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column with default k value of 200. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_union_theta_integer", c1, c2) + def kll_sketch_agg_double(columnName: String): Column = + kll_sketch_agg_double(Column(columnName)) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. - * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range + * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input + * sketch. + * + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = - tuple_union_theta_integer(Column(columnName1), Column(columnName2)) + def kll_merge_agg_bigint(e: Column, k: Column): Column = + Column.fn("kll_merge_agg_bigint", e, k) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range + * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) + def kll_merge_agg_bigint(e: Column, k: Int): Column = + Column.fn("kll_merge_agg_bigint", e, lit(k)) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range + * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_integer( - columnName1: String, - columnName2: String, - lgNomEntries: Int): Column = - tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries) + def kll_merge_agg_bigint(columnName: String, k: Int): Column = + kll_merge_agg_bigint(Column(columnName), k) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) + def kll_merge_agg_bigint(e: Column): Column = + Column.fn("kll_merge_agg_bigint", e) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_integer( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) + def kll_merge_agg_bigint(columnName: String): Column = + kll_merge_agg_bigint(Column(columnName)) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_theta_integer( - c1: Column, - c2: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_union_theta_integer", c1, c2, lgNomEntries, mode) + def kll_merge_agg_float(e: Column, k: Column): Column = + Column.fn("kll_merge_agg_float", e, k) /** - * Returns a string with human readable summary information about the KLL bigint sketch. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * * @param e - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def kll_sketch_to_string_bigint(e: Column): Column = - Column.fn("kll_sketch_to_string_bigint", e) + def kll_merge_agg_float(e: Column, k: Int): Column = + Column.fn("kll_merge_agg_float", e, lit(k)) /** - * Returns a string with human readable summary information about the KLL float sketch. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param e - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def kll_sketch_to_string_float(e: Column): Column = - Column.fn("kll_sketch_to_string_float", e) + def kll_merge_agg_float(columnName: String, k: Int): Column = + kll_merge_agg_float(Column(columnName), k) /** - * Returns a string with human readable summary information about the KLL double sketch. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * * @param e - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def kll_sketch_to_string_double(e: Column): Column = - Column.fn("kll_sketch_to_string_double", e) + def kll_merge_agg_float(e: Column): Column = + Column.fn("kll_merge_agg_float", e) /** - * Returns the number of items collected in the KLL bigint sketch. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param e - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def kll_sketch_get_n_bigint(e: Column): Column = - Column.fn("kll_sketch_get_n_bigint", e) + def kll_merge_agg_float(columnName: String): Column = + kll_merge_agg_float(Column(columnName)) /** - * Returns the number of items collected in the KLL float sketch. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * * @param e - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def kll_sketch_get_n_float(e: Column): Column = - Column.fn("kll_sketch_get_n_float", e) + def kll_merge_agg_double(e: Column, k: Column): Column = + Column.fn("kll_merge_agg_double", e, k) /** - * Returns the number of items collected in the KLL double sketch. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * * @param e - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def kll_sketch_get_n_double(e: Column): Column = - Column.fn("kll_sketch_get_n_double", e) + def kll_merge_agg_double(e: Column, k: Int): Column = + Column.fn("kll_merge_agg_double", e, lit(k)) /** - * Merges two KLL bigint sketch buffers together into one. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param left - * The first KLL bigint sketch. A column that evaluates to a binary. - * @param right - * The second KLL bigint sketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_merge_bigint(left: Column, right: Column): Column = - Column.fn("kll_sketch_merge_bigint", left, right) + def kll_merge_agg_double(columnName: String, k: Int): Column = + kll_merge_agg_double(Column(columnName), k) /** - * Merges two KLL float sketch buffers together into one. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param left - * The first KLL float sketch. A column that evaluates to a binary. - * @param right - * The second KLL float sketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_merge_float(left: Column, right: Column): Column = - Column.fn("kll_sketch_merge_float", left, right) + def kll_merge_agg_double(e: Column): Column = + Column.fn("kll_merge_agg_double", e) /** - * Merges two KLL double sketch buffers together into one. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param left - * The first KLL double sketch. A column that evaluates to a binary. - * @param right - * The second KLL double sketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_merge_double(left: Column, right: Column): Column = - Column.fn("kll_sketch_merge_double", left, right) + def kll_merge_agg_double(columnName: String): Column = + kll_merge_agg_double(Column(columnName)) /** - * Extracts a quantile value from a KLL bigint sketch given an input rank value. The rank can be - * a single value or an array. + * Aggregate function: returns the concatenation of non-null input values. * - * @param sketch - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @param rank - * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or - * an array. Must be a constant. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * The target column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long, or an array of longs when `rank` is an array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = - Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) + def listagg(e: Column): Column = Column.fn("listagg", e) /** - * Extracts a quantile value from a KLL float sketch given an input rank value. The rank can be - * a single value or an array. + * Aggregate function: returns the concatenation of non-null input values, separated by the + * delimiter. * - * @param sketch - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @param rank - * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or - * an array. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * The target column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * The delimiter used to separate the values. A column that evaluates to a string or binary. + * Must be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a float, or an array of floats when `rank` is an array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = - Column.fn("kll_sketch_get_quantile_float", sketch, rank) + def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) /** - * Extracts a quantile value from a KLL double sketch given an input rank value. The rank can be - * a single value or an array. + * Aggregate function: returns the concatenation of distinct non-null input values. * - * @param sketch - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @param rank - * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or - * an array. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `rank` is an - * array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = - Column.fn("kll_sketch_get_quantile_double", sketch, rank) + def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) /** - * Extracts a rank value from a KLL bigint sketch given an input quantile value. The quantile - * can be a single value or an array. + * Aggregate function: returns the concatenation of distinct non-null input values, separated by + * the delimiter. * - * @param sketch - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @param quantile - * The quantile value(s) to lookup. A column that evaluates to an integral or an array. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an - * array. + * Returns a column that evaluates to a string. */ - def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = - Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) + def listagg_distinct(e: Column, delimiter: Column): Column = + Column.fn("listagg", isDistinct = true, e, delimiter) /** - * Extracts a rank value from a KLL float sketch given an input quantile value. The quantile can - * be a single value or an array. + * Aggregate function: returns the concatenation of non-null input values. Alias for `listagg`. * - * @param sketch - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @param quantile - * The quantile value(s) to lookup. A column that evaluates to a numeric or an array. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an - * array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = - Column.fn("kll_sketch_get_rank_float", sketch, quantile) + def string_agg(e: Column): Column = Column.fn("string_agg", e) /** - * Extracts a rank value from a KLL double sketch given an input quantile value. The quantile - * can be a single value or an array. + * Aggregate function: returns the concatenation of non-null input values, separated by the + * delimiter. Alias for `listagg`. * - * @param sketch - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @param quantile - * The quantile value(s) to look up. A column that evaluates to a numeric or an array. Must be - * a constant. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an - * array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = - Column.fn("kll_sketch_get_rank_double", sketch, quantile) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // DateTime functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) /** - * Returns the date that is `numMonths` after `startDate`. + * Aggregate function: returns the concatenation of distinct non-null input values. Alias for + * `listagg`. * - * @param startDate - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param numMonths - * The number of months to add to `startDate`, can be negative to subtract months. A column - * that evaluates to an integer. + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * A date, or null if `startDate` was a string that could not be cast to a date. Returns a - * column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a string. */ - def add_months(startDate: Column, numMonths: Int): Column = - add_months(startDate, lit(numMonths)) + def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) /** - * Returns the date that is `numMonths` after `startDate`. - * - * @param startDate - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param numMonths - * A column of the number of months to add to `startDate`, can be negative to subtract months. - * A column that evaluates to an integer. + * Aggregate function: returns the concatenation of distinct non-null input values, separated by + * the delimiter. Alias for `listagg`. + * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * A date, or null if `startDate` was a string that could not be cast to a date. Returns a - * column that evaluates to a date. - * @group datetime_funcs - * @since 3.0.0 + * Returns a column that evaluates to a string. */ - def add_months(startDate: Column, numMonths: Column): Column = - Column.fn("add_months", startDate, numMonths) + def string_agg_distinct(e: Column, delimiter: Column): Column = + Column.fn("string_agg", isDistinct = true, e, delimiter) /** - * Returns the current date at the start of query evaluation as a date column. All calls of - * current_date within the same query return the same value. + * Aggregate function: alias for `var_samp`. * - * @group datetime_funcs - * @since 3.5.0 + * @param e + * the column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a double. */ - def curdate(): Column = Column.fn("curdate") + def variance(e: Column): Column = Column.fn("variance", e) /** - * Returns the current date at the start of query evaluation as a date column. All calls of - * current_date within the same query return the same value. + * Aggregate function: alias for `var_samp`. * - * @group datetime_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a double. */ - def current_date(): Column = Column.fn("current_date") + def variance(columnName: String): Column = variance(Column(columnName)) /** - * Returns the current session local timezone. + * Aggregate function: returns the unbiased variance of the values in a group. * - * @group datetime_funcs - * @since 3.5.0 + * @param e + * the column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def current_timezone(): Column = Column.fn("current_timezone") + def var_samp(e: Column): Column = Column.fn("var_samp", e) /** - * Returns the current timestamp at the start of query evaluation as a timestamp column. All - * calls of current_timestamp within the same query return the same value. + * Aggregate function: returns the unbiased variance of the values in a group. * - * @group datetime_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def current_timestamp(): Column = Column.fn("current_timestamp") + def var_samp(columnName: String): Column = var_samp(Column(columnName)) /** - * Returns the current timestamp at the start of query evaluation. + * Aggregate function: returns the population variance of the values in a group. * - * @group datetime_funcs - * @since 3.5.0 + * @param e + * the column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def now(): Column = Column.fn("now") + def var_pop(e: Column): Column = Column.fn("var_pop", e) /** - * Returns the current timestamp without time zone at the start of query evaluation as a - * timestamp without time zone column. All calls of localtimestamp within the same query return - * the same value. + * Aggregate function: returns the population variance of the values in a group. * - * @group datetime_funcs - * @since 3.3.0 + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def localtimestamp(): Column = Column.fn("localtimestamp") + def var_pop(columnName: String): Column = var_pop(Column(columnName)) /** - * Converts a date/timestamp/string to a value of string in the format specified by the date - * format given by the second argument. - * - * See Datetime - * Patterns for valid date and time format patterns + * Aggregate function: returns the average of the independent variable for non-null pairs in a + * group, where `y` is the dependent variable and `x` is the independent variable. * - * @param dateExpr - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp or time. - * @param format - * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to - * a string. + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns - * a column that evaluates to a string. - * @note - * Use specialized functions like [[year]] whenever possible as they benefit from a - * specialized implementation. - * @throws IllegalArgumentException - * if the `format` pattern is invalid - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def date_format(dateExpr: Column, format: String): Column = - Column.fn("date_format", dateExpr, lit(format)) + def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) /** - * Returns the date that is `days` days after `start` + * Aggregate function: returns the average of the dependent variable for non-null pairs in a + * group, where `y` is the dependent variable and `x` is the independent variable. * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * The number of days to add to `start`, can be negative to subtract days. A column that - * evaluates to an integer, short, or byte. + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def date_add(start: Column, days: Int): Column = date_add(start, lit(days)) + def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) /** - * Returns the date that is `days` days after `start` + * Aggregate function: returns the number of non-null number pairs in a group, where `y` is the + * dependent variable and `x` is the independent variable. * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * A column of the number of days to add to `start`, can be negative to subtract days. A - * column that evaluates to an integer, short, or byte. + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 3.0.0 + * Returns a column that evaluates to a long. */ - def date_add(start: Column, days: Column): Column = Column.fn("date_add", start, days) + def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) /** - * Returns the date that is `days` days after `start` + * Aggregate function: returns the intercept of the univariate linear regression line for + * non-null pairs in a group, where `y` is the dependent variable and `x` is the independent + * variable. * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * A column of the number of days to add to `start`, can be negative to subtract days. A - * column that evaluates to an integer, short, or byte. - * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ - def dateadd(start: Column, days: Column): Column = Column.fn("dateadd", start, days) + def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) /** - * Returns the date that is `days` days before `start` + * Aggregate function: returns the coefficient of determination for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * The number of days to subtract from `start`, can be negative to add days. A column that - * evaluates to an integer, short, or byte. + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def date_sub(start: Column, days: Int): Column = date_sub(start, lit(days)) + def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) /** - * Returns the date that is `days` days before `start` + * Aggregate function: returns the slope of the linear regression line for non-null pairs in a + * group, where `y` is the dependent variable and `x` is the independent variable. * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * A column of the number of days to subtract from `start`, can be negative to add days. A - * column that evaluates to an integer, short, or byte. + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 3.0.0 + * Returns a column that evaluates to a double. */ - def date_sub(start: Column, days: Column): Column = - Column.fn("date_sub", start, days) + def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) /** - * Returns the number of days from `start` to `end`. - * - * Only considers the date part of the input. For example: - * {{{ - * datediff("2018-01-10 00:00:00", "2018-01-09 23:59:59") - * // returns 1 - * }}} - * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. + * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. + * + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def datediff(end: Column, start: Column): Column = Column.fn("datediff", end, start) + def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) /** - * Returns the number of days from `start` to `end`. - * - * Only considers the date part of the input. For example: - * {{{ - * date_diff("2018-01-10 00:00:00", "2018-01-09 23:59:59") - * // returns 1 - * }}} + * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @return - * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. - * @group datetime_funcs + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ - def date_diff(end: Column, start: Column): Column = Column.fn("date_diff", end, start) + def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) /** - * Create date from the number of `days` since 1970-01-01. + * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. * - * @param days - * The number of days since 1970-01-01. A column that evaluates to an integral. - * @group datetime_funcs + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a double. */ - def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) + def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) /** - * Extracts the year as an integer from a given date/timestamp/string. + * Aggregate function: returns some value of `e` for a group of rows. + * * @param e - * The date, timestamp or string to extract the year from. A column that evaluates to a date, - * timestamp or string. + * The column to return some value from. A column of any type. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column of the same type as the input. */ - def year(e: Column): Column = Column.fn("year", e) + def any_value(e: Column): Column = Column.fn("any_value", e) /** - * Extracts the quarter as an integer from a given date/timestamp/string. + * Aggregate function: returns some value of `e` for a group of rows. If `ignoreNulls` is true, + * returns only non-null values. + * * @param e - * The date, timestamp or string to extract the quarter from. A column that evaluates to a - * date, timestamp or string. + * The column to return some value from. A column of any type. + * @param ignoreNulls + * If true, returns only non-null values. A column that evaluates to a boolean. Must be a + * constant. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column of the same type as the input. */ - def quarter(e: Column): Column = Column.fn("quarter", e) + def any_value(e: Column, ignoreNulls: Column): Column = + Column.fn("any_value", e, ignoreNulls) /** - * Extracts the month as an integer from a given date/timestamp/string. + * Aggregate function: returns the number of `TRUE` values for the expression. + * * @param e - * The date, timestamp or string to extract the month from. A column that evaluates to a date, - * timestamp or string. + * The expression to count TRUE values of. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a long. */ - def month(e: Column): Column = Column.fn("month", e) + def count_if(e: Column): Column = Column.fn("count_if", e) /** - * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 - * for a Sunday through to 7 for a Saturday + * Aggregate function: computes a histogram on numeric 'expr' using nb bins. The return value is + * an array of (x,y) pairs representing the centers of the histogram's bins. As the value of + * 'nb' is increased, the histogram approximation gets finer-grained, but may yield artifacts + * around outliers. In practice, 20-40 histogram bins appear to work well, with more bins being + * required for skewed or smaller datasets. Note that this function creates a histogram with + * non-uniform bin widths. It offers no guarantees in terms of the mean-squared-error of the + * histogram, but in practice is comparable to the histograms produced by the R/S-Plus + * statistical computing packages. Note: the output type of the 'x' field in the return value is + * propagated from the input value consumed in the aggregate function. + * * @param e - * The date, timestamp or string to extract the day of the week from. A column that evaluates - * to a date, timestamp or string. + * The column to compute the histogram on. A column that evaluates to a numeric. + * @param nBins + * The number of histogram bins. A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 2.3.0 + * Returns a column that evaluates to an array. */ - def dayofweek(e: Column): Column = Column.fn("dayofweek", e) + def histogram_numeric(e: Column, nBins: Column): Column = + Column.fn("histogram_numeric", e, nBins) /** - * Extracts the day of the month as an integer from a given date/timestamp/string. + * Aggregate function: returns true if all values of `e` are true. + * * @param e - * The date, timestamp or string to extract the day of the month from. A column that evaluates - * to a date, timestamp or string. + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a boolean. */ - def dayofmonth(e: Column): Column = Column.fn("dayofmonth", e) + def every(e: Column): Column = Column.fn("every", e) /** - * Extracts the day of the month as an integer from a given date/timestamp/string. + * Aggregate function: returns true if all values of `e` are true. + * * @param e - * The date, timestamp or string to extract the day of the month from. A column that evaluates - * to a date, timestamp or string. - * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ - def day(e: Column): Column = Column.fn("day", e) + def bool_and(e: Column): Column = Column.fn("bool_and", e) /** - * Extracts the day of the year as an integer from a given date/timestamp/string. + * Aggregate function: returns true if at least one value of `e` is true. + * * @param e - * The date, timestamp or string to extract the day of the year from. A column that evaluates - * to a date, timestamp or string. + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a boolean. */ - def dayofyear(e: Column): Column = Column.fn("dayofyear", e) + def some(e: Column): Column = Column.fn("some", e) /** - * Extracts the hours as an integer from a given date/time/timestamp/string. The input may also - * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in - * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * Aggregate function: returns true if at least one value of `e` is true. + * * @param e - * The column to extract the hours from. A column that evaluates to a date, time, timestamp or - * string. + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a boolean. */ - def hour(e: Column): Column = Column.fn("hour", e) + def any(e: Column): Column = Column.fn("any", e) /** - * Extracts a part of the date/timestamp or interval source. + * Aggregate function: returns true if at least one value of `e` is true. * - * @param field - * selects which part of the source should be extracted. - * @param source - * a date, time, timestamp or interval column from where `field` should be extracted. - * @return - * a part of the date/timestamp or interval source. Returns a column whose type depends on the - * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. - * @group datetime_funcs + * @param e + * column to check if at least one value is true. A column that evaluates to a boolean. + * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a boolean. */ - def extract(field: Column, source: Column): Column = { - Column.fn("extract", field, source) - } + def bool_or(e: Column): Column = Column.fn("bool_or", e) /** - * Extracts a part of the date/timestamp or interval source. + * Aggregate function: returns the bitwise AND of all non-null input values, or null if none. * - * @param field - * selects which part of the source should be extracted, and supported string values are as - * same as the fields of the equivalent function `extract`. - * @param source - * a date/timestamp or time or interval column from where `field` should be extracted. - * @return - * a part of the date/timestamp or interval source. Returns a column whose type depends on the - * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. - * @group datetime_funcs + * @param e + * target column to compute on. A column that evaluates to an integral. + * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ - def date_part(field: Column, source: Column): Column = { - Column.fn("date_part", field, source) - } + def bit_and(e: Column): Column = Column.fn("bit_and", e) /** - * Extracts a part of the date/timestamp or interval source. + * Aggregate function: returns the bitwise OR of all non-null input values, or null if none. * - * @param field - * selects which part of the source should be extracted, and supported string values are as - * same as the fields of the equivalent function `EXTRACT`. - * @param source - * a date/timestamp or interval column from where `field` should be extracted. - * @return - * a part of the date/timestamp or interval source. Returns a column whose type depends on the - * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. - * @group datetime_funcs + * @param e + * target column to compute on. A column that evaluates to an integral. + * @group agg_funcs * @since 3.5.0 + * @return + * Returns a column of the same type as the input. */ - def datepart(field: Column, source: Column): Column = { - Column.fn("datepart", field, source) - } + def bit_or(e: Column): Column = Column.fn("bit_or", e) /** - * Returns the last day of the month which the given date belongs to. For example, input - * "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015. + * Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. * * @param e - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. + * target column to compute on. A column that evaluates to an integral. + * @group agg_funcs + * @since 3.5.0 * @return - * A date, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column of the same type as the input. */ - def last_day(e: Column): Column = Column.fn("last_day", e) + def bit_xor(e: Column): Column = Column.fn("bit_xor", e) /** - * Extracts the minutes as an integer from a given date/time/timestamp/string. The input may - * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in - * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. + * Returns the mean calculated from values of a group and the result is null on overflow. + * * @param e - * The column to extract the minutes from. A column that evaluates to a date, time, timestamp - * or string. + * the value to compute the mean of. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def minute(e: Column): Column = Column.fn("minute", e) + def try_avg(e: Column): Column = Column.fn("try_avg", e) /** - * Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + * Returns the sum calculated from values of a group and the result is null on overflow. * * @param e - * The column to extract the day of the week from. A column that evaluates to a date, - * timestamp or string. - * @group datetime_funcs + * the value to compute the sum of. A column that evaluates to a numeric or interval. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a numeric. */ - def weekday(e: Column): Column = Column.fn("weekday", e) + def try_sum(e: Column): Column = Column.fn("try_sum", e) /** - * @param year - * The year to build the date. A column that evaluates to an integral. - * @param month - * The month to build the date. A column that evaluates to an integral. - * @param day - * The day to build the date. A column that evaluates to an integral. + * Returns a bitmap with the positions of the bits set from all the values from the input + * column. The input column will most likely be bitmap_bit_position(). + * + * @param col + * The input column will most likely be bitmap_bit_position(). A column that evaluates to an + * integral. + * @group agg_funcs + * @since 3.5.0 * @return - * A date created from year, month and day fields. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 3.3.0 + * Returns a column that evaluates to a binary. */ - def make_date(year: Column, month: Column, day: Column): Column = - Column.fn("make_date", year, month, day) + def bitmap_construct_agg(col: Column): Column = + Column.fn("bitmap_construct_agg", col) /** - * Returns number of months between dates `start` and `end`. - * - * A whole number is returned if both inputs have the same day of month or both are the last day - * of their respective months. Otherwise, the difference is calculated assuming 31 days per - * month. - * - * For example: - * {{{ - * months_between("2017-11-14", "2017-07-14") // returns 4.0 - * months_between("2017-01-01", "2017-01-10") // returns 0.29032258 - * months_between("2017-06-01", "2017-06-16 12:00:00") // returns -0.5 - * }}} + * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. The + * input column should be bitmaps created from bitmap_construct_agg(). * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can cast to a - * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * timestamp. + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. + * @group agg_funcs + * @since 3.5.0 * @return - * A double, or null if either `end` or `start` were strings that could not be cast to a - * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a - * double. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a binary. */ - def months_between(end: Column, start: Column): Column = - Column.fn("months_between", end, start) + def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) /** - * Returns number of months between dates `end` and `start`. If `roundOff` is set to true, the - * result is rounded off to 8 digits; it is not rounded otherwise. - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can cast to a - * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * timestamp. - * @param roundOff - * Whether to round off the result to 8 digits. A column that evaluates to a boolean. Must be - * a constant. - * @group datetime_funcs - * @since 2.4.0 + * Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. The + * input column should be bitmaps created from bitmap_construct_agg(). + * + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def months_between(end: Column, start: Column, roundOff: Boolean): Column = - Column.fn("months_between", end, start, lit(roundOff)) + def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) /** - * Returns the first date which is later than the value of the `date` column that is on the - * specified day of the week. - * - * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first - * Sunday after 2015-07-27. + * Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. The + * input column should be bitmaps created from bitmap_construct_agg(). * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param dayOfWeek - * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column - * that evaluates to a string. + * @param col + * A column containing bitmaps created by bitmap_construct_agg() and evaluating to binary + * data. + * @group agg_funcs + * @since 4.4.0 * @return - * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a binary. */ - def next_day(date: Column, dayOfWeek: String): Column = next_day(date, lit(dayOfWeek)) + def bitmap_xor_agg(col: Column): Column = Column.fn("bitmap_xor_agg", col) /** - * Returns the first date which is later than the value of the `date` column that is on the - * specified day of the week. - * - * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first - * Sunday after 2015-07-27. + * Aggregate function: returns a list of objects with duplicates. * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param dayOfWeek - * A column of the day of week. Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", - * "Fri", "Sat", "Sun". A column that evaluates to a string. + * @param e + * the input column. A column that evaluates to any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * @group agg_funcs + * @since 3.5.0 * @return - * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 3.2.0 + * Returns a column that evaluates to an array. */ - def next_day(date: Column, dayOfWeek: Column): Column = - Column.fn("next_day", date, dayOfWeek) + def array_agg(e: Column): Column = Column.fn("array_agg", e) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Window Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Extracts the seconds as an integer from a given date/time/timestamp/string. The input may - * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in - * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. - * @param e - * The column to extract the seconds from. A column that evaluates to a date, time, timestamp - * or string. + * Window function: computes the differences between consecutive cumulative counter values in a + * time series, thereby converting the counter from the cumulative to the delta format. + * + * Gracefully handles counter resets by returning NULL. Counter resets are detected when the + * counter value decreases. + * + * Use the PARTITION BY clause of the window to separate independent counters. This is done by + * specifying all columns which uniquely identify a time series. These are typically the counter + * name and any attributes tied to the counter. + * + * Use the ORDER BY clause of the window to order the observations by the associated timestamp + * in ascending order. + * + * @param value + * A cumulative counter. Must be a numeric data type. Must be non-negative. + * * @return - * An integer, or null if the input was a string that could not be cast to a timestamp. - * Returns a column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * The difference between the current and previous counter value within the window partition, + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. + * @group window_funcs + * @since 4.3.0 */ - def second(e: Column): Column = Column.fn("second", e) + def counter_diff(value: Column): Column = Column.fn("counter_diff", value) /** - * Extracts the week number as an integer from a given date/timestamp/string. + * Window function: computes the differences between consecutive cumulative counter values in a + * time series, thereby converting the counter from the cumulative to the delta format. * - * A week is considered to start on a Monday and week 1 is the first week with more than 3 days, - * as defined by ISO 8601 + * Gracefully handles counter resets by returning NULL. Counter resets are detected when the + * counter value decreases, or when the start time advances between rows. + * + * Use the PARTITION BY clause of the window to separate independent counters. This is done by + * specifying all columns which uniquely identify a time series. These are typically the counter + * name and any attributes tied to the counter. + * + * Use the ORDER BY clause of the window to order the observations by the associated timestamp + * in ascending order. + * + * @param value + * A cumulative counter. Must be a numeric data type. Must be non-negative. + * + * @param startTime + * A timestamp indicating when the counter was last set to zero. Used to signal counter + * resets. * - * @param e - * The column to extract the week number from. A column that evaluates to a date, timestamp or - * string. * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * The difference between the current and previous counter value within the window partition, + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. + * @group window_funcs + * @since 4.3.0 */ - def weekofyear(e: Column): Column = Column.fn("weekofyear", e) + def counter_diff(value: Column, startTime: Column): Column = + Column.fn("counter_diff", value, startTime) /** - * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string - * representing the timestamp of that moment in the current system time zone in the yyyy-MM-dd - * HH:mm:ss format. + * Window function: returns the cumulative distribution of values within a window partition, + * i.e. the fraction of rows that are below the current row. * - * @param ut - * A number of a type that is castable to a long, such as string or integer. Can be negative - * for timestamps before the unix epoch + * {{{ + * N = total number of rows in the partition + * cumeDist(x) = number of values before (and including) x / N + * }}} + * + * @group window_funcs + * @since 1.6.0 * @return - * A string, or null if the input was a string that could not be cast to a long. Returns a - * column that evaluates to a string. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def from_unixtime(ut: Column): Column = Column.fn("from_unixtime", ut) + def cume_dist(): Column = Column.fn("cume_dist") /** - * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string - * representing the timestamp of that moment in the current system time zone in the given - * format. + * Window function: returns the rank of rows within a window partition, without any gaps. * - * See Datetime - * Patterns for valid date and time format patterns + * The difference between rank and dense_rank is that denseRank leaves no gaps in ranking + * sequence when there are ties. That is, if you were ranking a competition using dense_rank and + * had three people tie for second place, you would say that all three were in second place and + * that the next person came in third. Rank would give me sequential numbers, making the person + * that came in third place (after the ties) would register as coming in fifth. * - * @param ut - * A number of a type that is castable to a long, such as string or integer. Can be negative - * for timestamps before the unix epoch - * @param f - * A date time pattern that the input will be formatted to + * This is equivalent to the DENSE_RANK function in SQL. + * + * @group window_funcs + * @since 1.6.0 * @return - * A string, or null if `ut` was a string that could not be cast to a long or `f` was an - * invalid date time pattern. Returns a column that evaluates to a string. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to an integer. */ - def from_unixtime(ut: Column, f: String): Column = - Column.fn("from_unixtime", ut, lit(f)) + def dense_rank(): Column = Column.fn("dense_rank") /** - * Returns the current Unix timestamp (in seconds) as a long. + * Window function: returns the value that is `offset` rows before the current row, and `null` + * if there is less than `offset` rows before the current row. For example, an `offset` of one + * will return the previous row at any given point in the window partition. * - * @note - * All calls of `unix_timestamp` within the same query return the same value (i.e. the current - * timestamp is calculated at the start of query evaluation). + * This is equivalent to the LAG function in SQL. * - * @group datetime_funcs - * @since 1.5.0 + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def unix_timestamp(): Column = unix_timestamp(current_timestamp()) + def lag(e: Column, offset: Int): Column = lag(e, offset, null) /** - * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds), using the - * default timezone and the default locale. + * Window function: returns the value that is `offset` rows before the current row, and `null` + * if there is less than `offset` rows before the current row. For example, an `offset` of one + * will return the previous row at any given point in the window partition. * - * @param s - * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` - * format + * This is equivalent to the LAG function in SQL. + * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * A long, or null if the input was a string not of the correct format. Returns a column that - * evaluates to a long. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column of the same type as the input. */ - def unix_timestamp(s: Column): Column = Column.fn("unix_timestamp", s) + def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) /** - * Converts time string with given pattern to Unix timestamp (in seconds). + * Window function: returns the value that is `offset` rows before the current row, and + * `defaultValue` if there is less than `offset` rows before the current row. For example, an + * `offset` of one will return the previous row at any given point in the window partition. * - * See Datetime - * Patterns for valid date and time format patterns + * This is equivalent to the LAG function in SQL. * - * @param s - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * string, date, or timestamp. - * @param p - * A date time pattern detailing the format of `s` when `s` is a string. A column that - * evaluates to a string. + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 * @return - * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid - * format. Returns a column that evaluates to a long. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column of the same type as the input. */ - def unix_timestamp(s: Column, p: String): Column = - Column.fn("unix_timestamp", s, lit(p)) + def lag(columnName: String, offset: Int, defaultValue: Any): Column = { + lag(Column(columnName), offset, defaultValue) + } /** - * Parses a string value to a time value. + * Window function: returns the value that is `offset` rows before the current row, and + * `defaultValue` if there is less than `offset` rows before the current row. For example, an + * `offset` of one will return the previous row at any given point in the window partition. * - * @param str - * A string to be parsed to time. A column that evaluates to a string. - * @return - * A time, or raises an error if the input is malformed. Returns a column that evaluates to a - * time. + * This is equivalent to the LAG function in SQL. * - * @group datetime_funcs - * @since 4.1.0 + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ - def to_time(str: Column): Column = { - Column.fn("to_time", str) + def lag(e: Column, offset: Int, defaultValue: Any): Column = { + lag(e, offset, defaultValue, false) } /** - * Parses a string value to a time value. + * Window function: returns the value that is `offset` rows before the current row, and + * `defaultValue` if there is less than `offset` rows before the current row. `ignoreNulls` + * determines whether null values of row are included in or eliminated from the calculation. For + * example, an `offset` of one will return the previous row at any given point in the window + * partition. * - * See Datetime - * Patterns for valid time format patterns. + * This is equivalent to the LAG function in SQL. * - * @param str - * A string to be parsed to time. - * @param format - * A time format pattern to follow. A column that evaluates to a string. + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @param ignoreNulls + * whether to ignore null values. A column that evaluates to a boolean. Must be a constant. + * @group window_funcs + * @since 3.2.0 * @return - * A time, or raises an error if the input is malformed. Returns a column that evaluates to a - * time. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column of the same type as the input. */ - def to_time(str: Column, format: Column): Column = { - Column.fn("to_time", str, format) - } + def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = + Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) /** - * Converts to a timestamp by casting rules to `TimestampType`. + * Window function: returns the value that is `offset` rows after the current row, and `null` if + * there is less than `offset` rows after the current row. For example, an `offset` of one will + * return the next row at any given point in the window partition. * - * @param s - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a string, date, timestamp, or numeric. + * This is equivalent to the LEAD function in SQL. + * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * A timestamp, or null if the input was a string that could not be cast to a timestamp. - * Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 2.2.0 + * Returns a column of the same type as the input. */ - def to_timestamp(s: Column): Column = Column.fn("to_timestamp", s) + def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } /** - * Converts time string with the given pattern to timestamp. + * Window function: returns the value that is `offset` rows after the current row, and `null` if + * there is less than `offset` rows after the current row. For example, an `offset` of one will + * return the next row at any given point in the window partition. * - * See Datetime - * Patterns for valid date and time format patterns + * This is equivalent to the LEAD function in SQL. * - * @param s - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a string, date, timestamp, or numeric. - * @param fmt - * A date time pattern detailing the format of `s` when `s` is a string. A column that - * evaluates to a string. + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was - * an invalid format. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 2.2.0 + * Returns a column of the same type as the input. */ - def to_timestamp(s: Column, fmt: String): Column = Column.fn("to_timestamp", s, lit(fmt)) + def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } /** - * Parses a string value to a time value. + * Window function: returns the value that is `offset` rows after the current row, and + * `defaultValue` if there is less than `offset` rows after the current row. For example, an + * `offset` of one will return the next row at any given point in the window partition. * - * @param str - * A string to be parsed to time. A column that evaluates to a string. - * @return - * A time, or null if the input is malformed. Returns a column that evaluates to a time. + * This is equivalent to the LEAD function in SQL. * - * @group datetime_funcs - * @since 4.1.0 + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 + * @return + * Returns a column of the same type as the input. */ - def try_to_time(str: Column): Column = { - Column.fn("try_to_time", str) + def lead(columnName: String, offset: Int, defaultValue: Any): Column = { + lead(Column(columnName), offset, defaultValue) } /** - * Parses a string value to a time value. + * Window function: returns the value that is `offset` rows after the current row, and + * `defaultValue` if there is less than `offset` rows after the current row. For example, an + * `offset` of one will return the next row at any given point in the window partition. * - * See Datetime - * Patterns for valid time format patterns. + * This is equivalent to the LEAD function in SQL. * - * @param str - * A string to be parsed to time. - * @param format - * A time format pattern to follow. A column that evaluates to a string. + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 * @return - * A time, or null if the input is malformed. Returns a column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column of the same type as the input. */ - def try_to_time(str: Column, format: Column): Column = { - Column.fn("try_to_time", str, format) + def lead(e: Column, offset: Int, defaultValue: Any): Column = { + lead(e, offset, defaultValue, false) } /** - * Parses the `s` with the `format` to a timestamp. The function always returns null on an - * invalid input with`/`without ANSI SQL mode enabled. The result data type is consistent with - * the value of configuration `spark.sql.timestampType`. + * Window function: returns the value that is `offset` rows after the current row, and + * `defaultValue` if there is less than `offset` rows after the current row. `ignoreNulls` + * determines whether null values of row are included in or eliminated from the calculation. The + * default value of `ignoreNulls` is false. For example, an `offset` of one will return the next + * row at any given point in the window partition. * - * @param s - * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. - * @param format - * Format to use to convert timestamp values. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * This is equivalent to the LEAD function in SQL. + * + * @param e + * The column to compute the lead value for. A column of any type. + * @param offset + * Number of rows after the current row to look ahead. A column that evaluates to an integral. + * Must be a constant. + * @param defaultValue + * Value to return when there are fewer than `offset` rows after the current row. A column of + * any type. Must be a constant. + * @param ignoreNulls + * Whether to skip null values when computing the result. A column that evaluates to a + * boolean. Must be a constant. + * @group window_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column of the same type as the input. */ - def try_to_timestamp(s: Column, format: Column): Column = - Column.fn("try_to_timestamp", s, format) + def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = + Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) /** - * Parses the `s` to a timestamp. The function always returns null on an invalid input - * with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The result - * data type is consistent with the value of configuration `spark.sql.timestampType`. + * Window function: returns the value that is the `offset`th row of the window frame (counting + * from 1), and `null` if the size of window frame is less than `offset` rows. * - * @param s - * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. - * @group datetime_funcs - * @since 3.5.0 + * It will return the `offset`th non-null value it sees when ignoreNulls is set to true. If all + * values are null, then null is returned. + * + * This is equivalent to the nth_value function in SQL. + * + * @param e + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. + * @param ignoreNulls + * Whether the nth value should skip nulls when determining which row to use. A column that + * evaluates to a boolean. Must be a constant. + * @group window_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column of the same type as the input. */ - def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) + def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = + Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) /** - * Converts the column into `DateType` by casting rules to `DateType`. + * Window function: returns the value that is the `offset`th row of the window frame (counting + * from 1), and `null` if the size of window frame is less than `offset` rows. + * + * This is equivalent to the nth_value function in SQL. * * @param e - * Input column of values to convert. A column that evaluates to a string, date, or timestamp. - * @group datetime_funcs - * @since 1.5.0 + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. + * @group window_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a date. + * Returns a column of the same type as the input. */ - def to_date(e: Column): Column = Column.fn("to_date", e) + def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) /** - * Converts the column into a `DateType` with a specified format + * Window function: returns the ntile group id (from 1 to `n` inclusive) in an ordered window + * partition. For example, if `n` is 4, the first quarter of the rows will get value 1, the + * second quarter will get 2, the third quarter will get 3, and the last quarter will get 4. * - * See Datetime - * Patterns for valid date and time format patterns + * This is equivalent to the NTILE function in SQL. * - * @param e - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * string, date, or timestamp. - * @param fmt - * A date time pattern detailing the format of `e` when `e`is a string. A column that - * evaluates to a string. + * @param n + * The number of groups to divide the window partition into. A column that evaluates to an + * integral. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an - * invalid format. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 2.2.0 + * Returns a column that evaluates to an integer. */ - def to_date(e: Column, fmt: String): Column = Column.fn("to_date", e, lit(fmt)) + def ntile(n: Int): Column = Column.fn("ntile", lit(n)) /** - * This is a special version of `to_date` that performs the same operation, but returns a NULL - * value instead of raising an error if date cannot be created. + * Window function: returns the relative rank (i.e. percentile) of rows within a window + * partition. * - * @param e - * Input column of values to convert. A column that evaluates to a string, date, or timestamp. - * @group datetime_funcs - * @since 4.1.0 + * This is computed by: + * {{{ + * (rank of row in its partition - 1) / (number of rows in the partition - 1) + * }}} + * + * This is equivalent to the PERCENT_RANK function in SQL. + * + * @group window_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a double. */ - def try_to_date(e: Column): Column = Column.fn("try_to_date", e) + def percent_rank(): Column = Column.fn("percent_rank") /** - * This is a special version of `to_date` that performs the same operation, but returns a NULL - * value instead of raising an error if date cannot be created. + * Window function: returns the rank of rows within a window partition. * - * @param e - * Input column of values to convert. A column that evaluates to a string, date, or timestamp. - * @param fmt - * Format to use to convert date values. A column that evaluates to a string. Must be a - * constant. - * @group datetime_funcs - * @since 4.1.0 + * The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking + * sequence when there are ties. That is, if you were ranking a competition using dense_rank and + * had three people tie for second place, you would say that all three were in second place and + * that the next person came in third. Rank would give me sequential numbers, making the person + * that came in third place (after the ties) would register as coming in fifth. + * + * This is equivalent to the RANK function in SQL. + * + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to an integer. */ - def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) + def rank(): Column = Column.fn("rank") /** - * Returns the number of days since 1970-01-01. + * Window function: returns a sequential number starting at 1 within a window partition. * - * @param e - * Input column of values to convert. A column that evaluates to a date. - * @group datetime_funcs - * @since 3.5.0 + * @group window_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to an integer. */ - def unix_date(e: Column): Column = Column.fn("unix_date", e) + def row_number(): Column = Column.fn("row_number") + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Generator Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Returns the number of microseconds since 1970-01-01 00:00:00 UTC. + * Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default + * unless specified otherwise. * - * @param e - * Input column of values to convert. A column that evaluates to a timestamp. - * @group datetime_funcs + * @param cols + * The first column must be a constant integer for the number of rows, and the remaining + * columns are the input elements to be separated into rows. + * @group generator_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def unix_micros(e: Column): Column = Column.fn("unix_micros", e) + @scala.annotation.varargs + def stack(cols: Column*): Column = Column.fn("stack", cols: _*) /** - * Returns the number of nanoseconds since 1970-01-01 00:00:00 UTC for a nanosecond-precision - * timestamp (`TIMESTAMP_LTZ(p)` / `TIMESTAMP_NTZ(p)`, `p` in `[7, 9]`). The result is a - * lossless `DECIMAL(21, 0)`. + * Creates a new row for each element in the given array or map column. Uses the default column + * name `col` for elements in the array and `key` and `value` for elements in the map unless + * specified otherwise. * * @param e - * input column of nanosecond-precision timestamp values to convert. A column that evaluates - * to a timestamp. - * @group datetime_funcs - * @since 4.3.0 + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ - def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) + def explode(e: Column): Column = Column.fn("explode", e) /** - * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of - * precision. + * Creates a new row for each element in the given array or map column. Uses the default column + * name `col` for elements in the array and `key` and `value` for elements in the map unless + * specified otherwise. Unlike explode, if the array/map is null or empty then null is produced. * * @param e - * input column of values to convert. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 2.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ - def unix_millis(e: Column): Column = Column.fn("unix_millis", e) + def explode_outer(e: Column): Column = Column.fn("explode_outer", e) /** - * Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of - * precision. + * Creates a new row for each element with position in the given array or map column. Uses the + * default column name `pos` for position, and `col` for elements in the array and `key` and + * `value` for elements in the map unless specified otherwise. * * @param e - * input column of values to convert. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a long. + * Returns the position column and a column of the element type of the input array, or the + * position column and the key and value columns of the input map. */ - def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) + def posexplode(e: Column): Column = Column.fn("posexplode", e) /** - * Returns date truncated to the unit specified by the format. - * - * For example, `trunc("2018-11-19 12:01:19", "year")` returns 2018-01-01 - * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param format: - * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - * Other options are: 'week', 'quarter'. A column that evaluates to a string. + * Creates a new row for each element with position in the given array or map column. Uses the + * default column name `pos` for position, and `col` for elements in the array and `key` and + * `value` for elements in the map unless specified otherwise. Unlike posexplode, if the + * array/map is null or empty then the row (null, null) is produced. * + * @param e + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 2.2.0 * @return - * A date, or null if `date` was a string that could not be cast to a date or `format` was an - * invalid value. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns the position column and a column of the element type of the input array, or the + * position column and the key and value columns of the input map. */ - def trunc(date: Column, format: String): Column = Column.fn("trunc", date, lit(format)) + def posexplode_outer(e: Column): Column = Column.fn("posexplode_outer", e) /** - * Returns timestamp truncated to the unit specified by the format. - * - * For example, `date_trunc("year", "2018-11-19 12:01:19")` returns 2018-01-01 00:00:00 + * Creates a new row for each element in the given array of structs. * - * @param format: - * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', - * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', - * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. - * @param timestamp - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. + * @param e + * the target column to explode. A column that evaluates to an array of structs. + * @group generator_funcs + * @since 3.4.0 * @return - * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or - * `format` was an invalid value. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 2.3.0 + * Returns a column that evaluates to a struct. */ - def date_trunc(format: String, timestamp: Column): Column = - Column.fn("date_trunc", lit(format), timestamp) + def inline(e: Column): Column = Column.fn("inline", e) /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders - * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 - * 03:40:00.0'. + * Creates a new row for each element in the given array of structs. Unlike inline, if the array + * is null or empty then null is produced for each nested column. * - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. It should be in - * the format of either region-based zone IDs or zone offsets. Region IDs must have the form - * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format - * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A - * column that evaluates to a string. + * @param e + * the target column to explode. A column that evaluates to an array of structs. + * @group generator_funcs + * @since 3.4.0 * @return - * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a struct. */ - def from_utc_timestamp(ts: Column, tz: String): Column = from_utc_timestamp(ts, lit(tz)) + def inline_outer(e: Column): Column = Column.fn("inline_outer", e) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Partition Transformation Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders - * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 - * 03:40:00.0'. - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. A column that - * evaluates to a string. - * @group datetime_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to a timestamp. + * (Java-specific) A transform for timestamps and dates to partition data into years. + * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def from_utc_timestamp(ts: Column, tz: Column): Column = - Column.fn("from_utc_timestamp", ts, tz) + def years(e: Column): Column = partitioning.years(e) /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time - * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield - * '2017-07-14 01:40:00.0'. + * (Java-specific) A transform for timestamps and dates to partition data into months. * - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. It should be in - * the format of either region-based zone IDs or zone offsets. Region IDs must have the form - * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format - * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A - * column that evaluates to a string. - * @return - * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 1.5.0 + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def to_utc_timestamp(ts: Column, tz: String): Column = to_utc_timestamp(ts, lit(tz)) + def months(e: Column): Column = partitioning.months(e) /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time - * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield - * '2017-07-14 01:40:00.0'. - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. A column that - * evaluates to a string. - * @group datetime_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to a timestamp. + * (Java-specific) A transform for timestamps and dates to partition data into days. + * + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) + def days(e: Column): Column = partitioning.days(e) /** - * Bucketize rows into one or more time windows given a timestamp specifying column. Window - * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window - * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in - * the order of months are not supported. The following example takes the average stock price - * for a one minute window every 10 seconds starting 5 seconds after the hour: - * - * {{{ - * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType - * df.groupBy(window($"timestamp", "1 minute", "10 seconds", "5 seconds"), $"stockId") - * .agg(mean("price")) - * }}} - * - * The windows will look like: - * - * {{{ - * 09:00:05-09:01:05 - * 09:00:15-09:01:15 - * 09:00:25-09:01:25 ... - * }}} - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. - * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param windowDuration - * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that - * the duration is a fixed length of time, and does not vary over time according to a - * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A - * column that evaluates to a string. - * @param slideDuration - * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will - * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. - * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This - * duration is likewise absolute, and does not vary according to a calendar. A column that - * evaluates to a string. - * @param startTime - * The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals. - * For example, in order to have hourly tumbling windows that start 15 minutes past the hour, - * e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`. A column that - * evaluates to a string. + * (Java-specific) A transform for timestamps to partition data into hours. * - * @group datetime_funcs - * @since 2.0.0 - * @return - * Returns a column that evaluates to a struct. + * @param e + * target date or timestamp column to work on. A column that evaluates to a date or timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def window( - timeColumn: Column, - windowDuration: String, - slideDuration: String, - startTime: String): Column = - Column.fn("window", timeColumn, lit(windowDuration), lit(slideDuration), lit(startTime)) + def hours(e: Column): Column = partitioning.hours(e) /** - * Bucketize rows into one or more time windows given a timestamp specifying column. Window - * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window - * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in - * the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 - * UTC. The following example takes the average stock price for a one minute window every 10 - * seconds: - * - * {{{ - * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType - * df.groupBy(window($"timestamp", "1 minute", "10 seconds"), $"stockId") - * .agg(mean("price")) - * }}} - * - * The windows will look like: - * - * {{{ - * 09:00:00-09:01:00 - * 09:00:10-09:01:10 - * 09:00:20-09:01:20 ... - * }}} - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. + * (Java-specific) A transform for any type that partitions by a hash of the input column. * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param windowDuration - * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that - * the duration is a fixed length of time, and does not vary over time according to a - * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A - * column that evaluates to a string. - * @param slideDuration - * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will - * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. - * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This - * duration is likewise absolute, and does not vary according to a calendar. A column that - * evaluates to a string. + * @param numBuckets + * The number of buckets. A column that evaluates to an integral. Must be a constant. + * @param e + * The input column to partition. A column of any type. + * @group partition_transforms + * @since 3.0.0 + */ + def bucket(numBuckets: Column, e: Column): Column = partitioning.bucket(numBuckets, e) + + /** + * (Java-specific) A transform for any type that partitions by a hash of the input column. * - * @group datetime_funcs - * @since 2.0.0 - * @return - * Returns a column that evaluates to a struct. + * @param numBuckets + * The number of buckets. Must be a constant. + * @param e + * The input column to partition. A column of any type. + * @group partition_transforms + * @since 3.0.0 */ - def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { - window(timeColumn, windowDuration, slideDuration, "0 second") + def bucket(numBuckets: Int, e: Column): Column = partitioning.bucket(numBuckets, e) + + // scalastyle:off + // TODO(SPARK-45970): Use @static annotation so Java can access to those + // API in the same way. Once we land this fix, should deprecate + // functions.hours, days, months, years and bucket. + object partitioning { + // scalastyle:on + /** + * (Scala-specific) A transform for timestamps and dates to partition data into years. + * + * @group partition_transforms + * @since 4.0.0 + */ + def years(e: Column): Column = Column.internalFn("years", e) + + /** + * (Scala-specific) A transform for timestamps and dates to partition data into months. + * + * @group partition_transforms + * @since 4.0.0 + */ + def months(e: Column): Column = Column.internalFn("months", e) + + /** + * (Scala-specific) A transform for timestamps and dates to partition data into days. + * + * @group partition_transforms + * @since 4.0.0 + */ + def days(e: Column): Column = Column.internalFn("days", e) + + /** + * (Scala-specific) A transform for timestamps to partition data into hours. + * + * @group partition_transforms + * @since 4.0.0 + */ + def hours(e: Column): Column = Column.internalFn("hours", e) + + /** + * (Scala-specific) A transform for any type that partitions by a hash of the input column. + * + * @group partition_transforms + * @since 4.0.0 + */ + def bucket(numBuckets: Column, e: Column): Column = Column.internalFn("bucket", numBuckets, e) + + /** + * (Scala-specific) A transform for any type that partitions by a hash of the input column. + * + * @group partition_transforms + * @since 4.0.0 + */ + def bucket(numBuckets: Int, e: Column): Column = bucket(lit(numBuckets), e) } + ////////////////////////////////////////////////////////////////////////////////////////////// + // CSV Functions + ////////////////////////////////////////////////////////////////////////////////////////////// + + // scalastyle:off line.size.limit /** - * Generates tumbling time windows given a timestamp specifying column. Window starts are - * inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) - * but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of - * months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC. The - * following example takes the average stock price for a one minute tumbling window: - * - * {{{ - * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType - * df.groupBy(window($"timestamp", "1 minute"), $"stockId") - * .agg(mean("price")) - * }}} - * - * The windows will look like: - * - * {{{ - * 09:00:00-09:01:00 - * 09:01:00-09:02:00 - * 09:02:00-09:03:00 ... - * }}} - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. + * Parses a column containing a CSV string into a `StructType` with the specified schema. + * Returns `null`, in the case of an unparseable string. * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param windowDuration - * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column - * that evaluates to a string. + * @param e + * a string column containing CSV data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the CSV string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the CSV is parsed. accepts the same options and the CSV data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. * - * @group datetime_funcs - * @since 2.0.0 + * @group csv_funcs + * @since 3.0.0 * @return * Returns a column that evaluates to a struct. */ - def window(timeColumn: Column, windowDuration: String): Column = { - window(timeColumn, windowDuration, windowDuration, "0 second") - } + // scalastyle:on line.size.limit + def from_csv(e: Column, schema: StructType, options: Map[String, String]): Column = + from_csv(e, lit(schema.toDDL), options.iterator) + // scalastyle:off line.size.limit /** - * Extracts the event time from the window column. - * - * The window column is of StructType { start: Timestamp, end: Timestamp } where start is - * inclusive and end is exclusive. Since event time can support microsecond precision, - * window_time(window) = window.end - 1 microsecond. + * (Java-specific) Parses a column containing a CSV string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @param windowColumn - * The window column (typically produced by window aggregation) of type StructType { start: - * Timestamp, end: Timestamp }. A column that evaluates to a struct. + * @param e + * a string column containing CSV data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the CSV string. A column that evaluates to a string. + * @param options + * options to control how the CSV is parsed. accepts the same options and the CSV data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. * - * @group datetime_funcs - * @since 3.4.0 + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a struct. */ - def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) + // scalastyle:on line.size.limit + def from_csv(e: Column, schema: Column, options: java.util.Map[String, String]): Column = + from_csv(e, schema, options.asScala.iterator) + + private def from_csv(e: Column, schema: Column, options: Iterator[(String, String)]): Column = + Column.fnWithOptions("from_csv", options, e, schema) /** - * Generates session window given a timestamp specifying column. - * - * Session window is one of dynamic windows, which means the length of window is varying - * according to the given inputs. The length of session window is defined as "the timestamp of - * latest input of the session + gap duration", so when the new inputs are bound to the current - * session window, the end time of session window can be expanded according to the new inputs. - * - * Windows can support microsecond precision. gapDuration in the order of months are not - * supported. - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. + * Parses a CSV string and infers its schema in DDL format. * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param gapDuration - * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column - * that evaluates to a string. + * @param csv + * a CSV string. A string. Must be a constant. * - * @group datetime_funcs - * @since 3.2.0 + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - def session_window(timeColumn: Column, gapDuration: String): Column = - session_window(timeColumn, lit(gapDuration)) + def schema_of_csv(csv: String): Column = schema_of_csv(lit(csv)) /** - * Generates session window given a timestamp specifying column. - * - * Session window is one of dynamic windows, which means the length of window is varying - * according to the given inputs. For static gap duration, the length of session window is - * defined as "the timestamp of latest input of the session + gap duration", so when the new - * inputs are bound to the current session window, the end time of session window can be - * expanded according to the new inputs. - * - * Besides a static gap duration value, users can also provide an expression to specify gap - * duration dynamically based on the input row. With dynamic gap duration, the closing of a - * session window does not depend on the latest input anymore. A session window's range is the - * union of all events' ranges which are determined by event start time and evaluated gap - * duration during the query execution. Note that the rows with negative or zero gap duration - * will be filtered out from the aggregation. - * - * Windows can support microsecond precision. gapDuration in the order of months are not - * supported. - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. + * Parses a CSV string and infers its schema in DDL format. * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param gapDuration - * A column specifying the timeout of the session. It could be static value, e.g. `10 - * minutes`, `1 second`, or an expression/UDF that specifies gap duration dynamically based on - * the input row. A column that evaluates to a string or interval. + * @param csv + * a foldable string column containing a CSV string. A column that evaluates to a string. * - * @group datetime_funcs - * @since 3.2.0 + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - def session_window(timeColumn: Column, gapDuration: Column): Column = - Column.fn("session_window", timeColumn, gapDuration) + def schema_of_csv(csv: Column): Column = schema_of_csv(csv, Collections.emptyMap()) + // scalastyle:off line.size.limit /** - * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. - * @param e - * unix time values. A column that evaluates to a numeric. - * @group datetime_funcs - * @since 3.1.0 + * Parses a CSV string and infers its schema in DDL format using options. + * + * @param csv + * a foldable string column containing a CSV string. A column that evaluates to a string. + * @param options + * options to control how the CSV is parsed. accepts the same options and the CSV data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * Returns a column that evaluates to a timestamp. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. + * @group csv_funcs + * @since 3.0.0 */ - def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) + // scalastyle:on line.size.limit + def schema_of_csv(csv: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("schema_of_csv", options.asScala.iterator, csv) + // scalastyle:off line.size.limit /** - * Creates timestamp from the number of milliseconds since UTC epoch. + * (Java-specific) Converts a column containing a `StructType` into a CSV string with the + * specified schema. Throws an exception, in the case of an unsupported type. * * @param e - * unix time values. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * a column containing a struct. A column that evaluates to a string. + * @param options + * options to control how the struct column is converted into a CSV string. It accepts the + * same options and the CSV data source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) + // scalastyle:on line.size.limit + def to_csv(e: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("to_csv", options.asScala.iterator, e) /** - * Creates timestamp from the number of microseconds since UTC epoch. + * Converts a column containing a `StructType` into a CSV string with the specified schema. + * Throws an exception, in the case of an unsupported type. * * @param e - * unix time values. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * a column containing a struct. A column that evaluates to a string. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) + def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // JSON Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Creates a timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from - * the number of nanoseconds since UTC epoch. + * Extracts json object from a json string based on json path specified, and returns json string + * of the extracted json object. It will return null if the input json string is invalid. * * @param e - * nanosecond values since the UTC epoch. A column that evaluates to an integral or decimal. - * @group datetime_funcs - * @since 4.3.0 + * the JSON string column. A column that evaluates to a string. + * @param path + * the JSON path to extract. A column that evaluates to a string. Must be a constant. + * @group json_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) + def get_json_object(e: Column, path: String): Column = + Column.fn("get_json_object", e, lit(path)) /** - * Gets the difference between the timestamps in the specified units by truncating the fraction - * part. + * Creates a new row for a json column according to the given field names. * - * @param unit - * the units of the difference between the given timestamps, e.g. 'YEAR', 'MONTH', 'DAY', - * 'HOUR'. A column that evaluates to a string. Must be a constant. - * @param start - * A timestamp which the expression subtracts from `end`. A column that evaluates to a - * timestamp. - * @param end - * A timestamp from which the expression subtracts `start`. A column that evaluates to a - * timestamp. - * @group datetime_funcs - * @since 4.0.0 + * @param json + * the JSON string column. A column that evaluates to a string. + * @param fields + * the field names to extract. A column that evaluates to a string. Must be a constant. + * @group json_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def timestamp_diff(unit: String, start: Column, end: Column): Column = - Column.internalFn("timestampdiff", lit(unit), start, end) + @scala.annotation.varargs + def json_tuple(json: Column, fields: String*): Column = { + require(fields.nonEmpty, "at least 1 field name should be given.") + Column.fn("json_tuple", json +: fields.map(lit): _*) + } + // scalastyle:off line.size.limit /** - * Adds the specified number of units to the given timestamp. + * (Scala-specific) Parses a column containing a JSON string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @param unit - * the units of datetime to add, e.g. 'YEAR', 'MONTH', 'DAY', 'HOUR'. A column that evaluates - * to a string. Must be a constant. - * @param quantity - * the number of units of time to add. A column that evaluates to an integral. - * @param ts - * A timestamp to which to add. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. Accepts the same options as the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a struct. */ - def timestamp_add(unit: String, quantity: Column, ts: Column): Column = - Column.internalFn("timestampadd", lit(unit), quantity, ts) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = + from_json(e, schema.asInstanceOf[DataType], options) + // scalastyle:off line.size.limit /** - * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets - * aligned to the default origin (1970-01-01 00:00:00). For `TIMESTAMP_NTZ`, bucketing is - * performed in UTC. For `TIMESTAMP`, year-month interval buckets and calendar-day components of - * day-time interval buckets align to the session time zone. + * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * - * @param bucketSize - * A day-time or year-month interval defining the bucket size. Must be positive and foldable. - * @param ts - * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. - * @group datetime_funcs - * @since 4.2.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def time_bucket(bucketSize: Column, ts: Column): Column = - Column.fn("time_bucket", bucketSize, ts) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { + from_json(e, lit(schema.sql), options.iterator) + } + // scalastyle:off line.size.limit /** - * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets - * aligned to `origin`. For `TIMESTAMP_NTZ`, bucketing is performed in UTC. For `TIMESTAMP`, - * year-month interval buckets and calendar-day components of day-time interval buckets align to - * the session time zone. + * (Java-specific) Parses a column containing a JSON string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @param bucketSize - * A day-time or year-month interval defining the bucket size. Must be positive and foldable. - * @param ts - * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. - * @param origin - * Alignment anchor. Must be the same type as `ts` and must be foldable. - * @group datetime_funcs - * @since 4.2.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a struct. */ - def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = - Column.fn("time_bucket", bucketSize, ts, origin) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = + from_json(e, schema, options.asScala.toMap) + // scalastyle:off line.size.limit /** - * Returns the difference between two times, measured in specified units. Throws a - * SparkIllegalArgumentException, in case the specified unit is not supported. + * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * - * @param unit - * A STRING representing the unit of the time difference. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A - * column that evaluates to a string. - * @param start - * A starting TIME. A column that evaluates to a time. - * @param end - * An ending TIME. A column that evaluates to a time. + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.2.0 * @return - * The difference between `end` and `start` times, measured in specified units. Returns a - * column that evaluates to a long. - * @note - * If any of the inputs is `NULL`, the result is `NULL`. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column of the type given by the schema (a struct, array, or map). */ - def time_diff(unit: Column, start: Column, end: Column): Column = { - Column.fn("time_diff", unit, start, end) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { + from_json(e, schema, options.asScala.toMap) } /** - * Returns `time` truncated to the `unit`. + * Parses a column containing a JSON string into a `StructType` with the specified schema. + * Returns `null`, in the case of an unparseable string. * - * @param unit - * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A - * column that evaluates to a string. - * @param time - * A TIME to truncate. A column that evaluates to a time. + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * A TIME truncated to the specified unit. Returns a column that evaluates to a time. - * @note - * If any of the inputs is `NULL`, the result is `NULL`. - * @throws IllegalArgumentException - * If the `unit` is not supported. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column that evaluates to a struct. */ - def time_trunc(unit: Column, time: Column): Column = { - Column.fn("time_trunc", unit, time) - } + def from_json(e: Column, schema: StructType): Column = + from_json(e, schema, Map.empty[String, String]) /** - * Creates a TIME from the number of seconds since midnight. + * Parses a column containing a JSON string into a `MapType` with `StringType` as keys type, + * `StructType` or `ArrayType` with the specified schema. Returns `null`, in the case of an + * unparseable string. * * @param e - * seconds since midnight (0 to 86399.999999). A column that evaluates to a numeric. - * @group datetime_funcs - * @since 4.2.0 + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * + * @group json_funcs + * @since 2.2.0 * @return - * Returns a column that evaluates to a time. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) + def from_json(e: Column, schema: DataType): Column = + from_json(e, schema, Map.empty[String, String]) + // scalastyle:off line.size.limit /** - * Creates a TIME from the number of milliseconds since midnight. + * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * * @param e - * milliseconds since midnight (0 to 86399999). A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.2.0 + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a time. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { + from_json(e, schema, options.asScala.toMap) + } + // scalastyle:off line.size.limit /** - * Creates a TIME from the number of microseconds since midnight. + * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * * @param e - * microseconds since midnight (0 to 86399999999). A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.2.0 + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to a time. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: String, options: Map[String, String]): Column = { + from_json(e, lit(schema), options.asJava) + } /** - * Extracts the number of seconds (including fractional seconds) from a TIME value. Returns a - * DECIMAL(14,6) to preserve microsecond precision. + * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns + * `null`, in the case of an unparseable string. * * @param e - * TIME value to convert. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.2.0 + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A column that evaluates to a string. + * + * @group json_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) + def from_json(e: Column, schema: Column): Column = { + from_json(e, schema, Map.empty[String, String].asJava) + } + // scalastyle:off line.size.limit /** - * Extracts the number of milliseconds since midnight from a TIME value. + * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns + * `null`, in the case of an unparseable string. * * @param e - * the TIME value to convert. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.2.0 + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A column that evaluates to a string. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { + from_json(e, schema, options.asScala.iterator) + } + + private def from_json( + e: Column, + schema: Column, + options: Iterator[(String, String)]): Column = { + Column.fnWithOptions("from_json", options, e, schema) + } /** - * Extracts the number of microseconds since midnight from a TIME value. + * Parses a JSON string and infers its schema in DDL format. * - * @param e - * the TIME value to convert. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.2.0 + * @param json + * a JSON string. A string. Must be a constant. + * + * @group json_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) + def schema_of_json(json: String): Column = schema_of_json(lit(json)) /** - * Parses the `timestamp` expression with the `format` expression to a timestamp with local time - * zone. Returns null with invalid input. + * Parses a JSON string and infers its schema in DDL format. * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @param format - * the format used to parse the timestamp values. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * @param json + * a foldable string column containing a JSON string. A column that evaluates to a string. + * + * @group json_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def to_timestamp_ltz(timestamp: Column, format: Column): Column = - Column.fn("to_timestamp_ltz", timestamp, format) + def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) + // scalastyle:off line.size.limit /** - * Parses the `timestamp` expression with the default format to a timestamp with local time - * zone. The default format follows casting rules to a timestamp. Returns null with invalid - * input. + * Parses a JSON string and infers its schema in DDL format using options. * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @group datetime_funcs - * @since 3.5.0 + * @param json + * a foldable string column containing JSON data. A column that evaluates to a string. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * Returns a column that evaluates to a timestamp. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. + * + * @group json_funcs + * @since 3.0.0 */ - def to_timestamp_ltz(timestamp: Column): Column = - Column.fn("to_timestamp_ltz", timestamp) + // scalastyle:on line.size.limit + def schema_of_json(json: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("schema_of_json", options.asScala.iterator, json) /** - * Parses the `timestamp_str` expression with the `format` expression to a timestamp without - * time zone. Returns null with invalid input. + * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of any + * other valid JSON string, `NULL` or an invalid JSON. * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @param format - * the format used to parse the timestamp values. A column that evaluates to a string. - * @group datetime_funcs + * @param e + * the JSON array string column. A column that evaluates to a string. + * @group json_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to an integer. */ - def to_timestamp_ntz(timestamp: Column, format: Column): Column = - Column.fn("to_timestamp_ntz", timestamp, format) + def json_array_length(e: Column): Column = Column.fn("json_array_length", e) /** - * Parses the `timestamp` expression with the default format to a timestamp without time zone. - * The default format follows casting rules to a timestamp. Returns null with invalid input. + * Returns all the keys of the outermost JSON object as an array. If a valid JSON object is + * given, all the keys of the outermost object will be returned as an array. If it is any other + * valid JSON string, an invalid JSON string or an empty string, the function returns null. * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @group datetime_funcs + * @param e + * the JSON object string column. A column that evaluates to a string. + * @group json_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to an array. */ - def to_timestamp_ntz(timestamp: Column): Column = - Column.fn("to_timestamp_ntz", timestamp) + def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) /** - * Returns the UNIX timestamp of the given time. + * Returns the type of the outermost JSON value as a string: one of 'object', 'array', 'string', + * 'number', 'boolean', or 'null'. Returns null for invalid or empty input. * - * @param timeExp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @param format - * the format used to convert the time values. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * the JSON string column. A column that evaluates to a string. + * @group json_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def to_unix_timestamp(timeExp: Column, format: Column): Column = - Column.fn("to_unix_timestamp", timeExp, format) + def json_typeof(e: Column): Column = Column.fn("json_typeof", e) + // scalastyle:off line.size.limit /** - * Returns the UNIX timestamp of the given time. + * (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into + * a JSON string with the specified schema. Throws an exception, in the case of an unsupported + * type. + * + * @param e + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. + * @param options + * options to control how the struct column is converted into a json string. accepts the same + * options and the json data source. See Data + * Source Option in the version you use. Additionally the function supports the `pretty` + * option which enables pretty JSON generation. A map of string options. Must be a constant. * - * @param timeExp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @group datetime_funcs - * @since 3.5.0 + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def to_unix_timestamp(timeExp: Column): Column = - Column.fn("to_unix_timestamp", timeExp) + // scalastyle:on line.size.limit + def to_json(e: Column, options: Map[String, String]): Column = + Column.fnWithOptions("to_json", options.iterator, e) + // scalastyle:off line.size.limit /** - * Extracts the three-letter abbreviated month name from a given date/timestamp/string. + * (Java-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into + * a JSON string with the specified schema. Throws an exception, in the case of an unsupported + * type. * - * @param timeExp - * the target date/timestamp to work on. A column that evaluates to a date, timestamp or - * string. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. + * @param options + * options to control how the struct column is converted into a json string. accepts the same + * options and the json data source. See Data + * Source Option in the version you use. Additionally the function supports the `pretty` + * option which enables pretty JSON generation. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return * Returns a column that evaluates to a string. */ - def monthname(timeExp: Column): Column = - Column.fn("monthname", timeExp) + // scalastyle:on line.size.limit + def to_json(e: Column, options: java.util.Map[String, String]): Column = + to_json(e, options.asScala.toMap) /** - * Extracts the three-letter abbreviated day name from a given date/timestamp/string. + * Converts a column containing a `StructType`, `ArrayType` or a `MapType` into a JSON string + * with the specified schema. Throws an exception, in the case of an unsupported type. * - * @param timeExp - * the target date/timestamp to work on. A column that evaluates to a date, timestamp or - * string. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. + * + * @group json_funcs + * @since 2.1.0 * @return * Returns a column that evaluates to a string. */ - def dayname(timeExp: Column): Column = - Column.fn("dayname", timeExp) + def to_json(e: Column): Column = + to_json(e, Map.empty[String, String]) ////////////////////////////////////////////////////////////////////////////////////////////// - // Collection functions + // VARIANT Functions ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Returns true if the array contains `value`, false if not. Returns null if the array or - * `value` is null, or if `value` is not found and the array contains a null element. - * @param column - * the target column containing the arrays. A column that evaluates to an array. - * @param value - * the value to check for in the array. A column that evaluates to a value matching the - * array's element type. - * @group array_funcs - * @since 1.5.0 - * @return - * Returns a column that evaluates to a boolean. - */ - def array_contains(column: Column, value: Any): Column = - Column.fn("array_contains", column, lit(value)) - - /** - * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. - * The new element/column is located at end of the ARRAY. + * Parses a JSON string and constructs a Variant value. Returns null if the input string is not + * a valid JSON value. * - * @param column - * the source column containing the array. A column that evaluates to an array. - * @param element - * the value to append to the array. A column that evaluates to a value matching the array's - * element type. - * @group array_funcs - * @since 3.4.0 - * @return - * Returns a column that evaluates to an array. - */ - def array_append(column: Column, element: Any): Column = - Column.fn("array_append", column, lit(element)) - - /** - * Returns `true` if `a1` and `a2` have at least one non-null element in common. If not and both - * the arrays are non-empty and any of them contains a `null`, it returns `null`. It returns - * `false` otherwise. - * @param a1 - * the first input array. A column that evaluates to an array. - * @param a2 - * the second input array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * @param json + * a string column that contains JSON data. A column that evaluates to a string. + * + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a variant. */ - def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) + def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) /** - * Returns an array containing all the elements in `x` from index `start` (or starting from the - * end if `start` is negative) with the specified `length`. - * - * @param x - * the array column to be sliced. A column that evaluates to an array. - * @param start - * the starting index. A column that evaluates to an integer. - * @param length - * the length of the slice. A column that evaluates to an integer. + * Parses a JSON string and constructs a Variant value. * - * @group array_funcs - * @since 2.4.0 + * @param json + * a string column that contains JSON data. A column that evaluates to a string. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a variant. */ - def slice(x: Column, start: Int, length: Int): Column = - slice(x, lit(start), lit(length)) + def parse_json(json: Column): Column = Column.fn("parse_json", json) /** - * Returns an array containing all the elements in `x` from index `start` (or starting from the - * end if `start` is negative) with the specified `length`. - * - * @param x - * the array column to be sliced. A column that evaluates to an array. - * @param start - * the starting index. A column that evaluates to an integer. - * @param length - * the length of the slice. A column that evaluates to an integer. + * Converts a column containing nested inputs (array/map/struct) into a variants where maps and + * structs are converted to variant objects which are unordered unlike SQL structs. Input maps + * can only have string keys. * - * @group array_funcs - * @since 3.1.0 + * @param col + * a column with a nested schema or column name. A column that evaluates to a struct, array, + * map, or variant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a variant. */ - def slice(x: Column, start: Column, length: Column): Column = - Column.fn("slice", x, start, length) + def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) /** - * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is - * negative or greater than the number of elements in the array. - * - * @param x - * the array column to be trimmed. A column that evaluates to an array. - * @param n - * the number of elements to remove from the end of the array. Must be between 0 and the - * number of elements in the array (inclusive). + * Creates a variant object from the given arrays of keys and values. The keys must be non-null + * strings and the two arrays must have the same length. * - * @group array_funcs + * @param keys + * a column that evaluates to an array of string keys. + * @param values + * a column that evaluates to an array of values. + * @group variant_funcs * @since 4.4.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a variant. */ - def trim_array(x: Column, n: Int): Column = trim_array(x, lit(n)) + def variant_from_arrays(keys: Column, values: Column): Column = + Column.fn("variant_from_arrays", keys, values) /** - * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is - * negative or greater than the number of elements in the array. - * - * @param x - * the array column to be trimmed. A column that evaluates to an array. - * @param n - * the number of elements to remove from the end of the array. Must be between 0 and the - * number of elements in the array (inclusive). + * Creates a variant object from an array of key/value struct entries. The keys must be non-null + * strings. * - * @group array_funcs + * @param entries + * a column that evaluates to an array of key/value structs. + * @group variant_funcs * @since 4.4.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a variant. */ - def trim_array(x: Column, n: Column): Column = Column.fn("trim_array", x, n) + def variant_from_entries(entries: Column): Column = + Column.fn("variant_from_entries", entries) /** - * Concatenates the elements of `column` using the `delimiter`. Null values are replaced with - * `nullReplacement`. - * @param column - * the input column containing the array. A column that evaluates to an array. - * @param delimiter - * the string used to join the array elements. A column that evaluates to a string. - * @param nullReplacement - * the string used to replace null values. A column that evaluates to a string. - * @group array_funcs - * @since 2.4.0 + * Check if a variant value is a variant null. Returns true if and only if the input is a + * variant null and false otherwise (including in the case of SQL NULL). + * + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a boolean. */ - def array_join(column: Column, delimiter: String, nullReplacement: String): Column = - Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) + def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) /** - * Concatenates the elements of `column` using the `delimiter`. - * @param column - * the input column containing the array. A column that evaluates to an array. - * @param delimiter - * the string used to join the array elements. A column that evaluates to a string. - * @group array_funcs - * @since 2.4.0 + * Check if a variant value is valid. Returns true if the variant is valid, false if it is + * malformed, and NULL if the input is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a boolean. */ - def array_join(column: Column, delimiter: String): Column = - Column.fn("array_join", column, lit(delimiter)) + def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) /** - * Concatenates multiple input columns together into a single column. The function works with - * strings, binary and compatible array columns. - * - * @param exprs - * Input columns to concatenate. A column that evaluates to a string, binary or an array. - * @note - * Returns null if any of the input columns are null. + * Removes fields or array elements from a variant at the given JSONPath locations. Multiple + * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. * - * @group collection_funcs - * @since 1.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the first JSONPath string. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A column that evaluates to a string. + * @param paths + * additional JSONPath arguments, applied after `path` in order. A column that evaluates to a + * string. + * @group variant_funcs + * @since 5.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a variant. */ @scala.annotation.varargs - def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) + def variant_delete(v: Column, path: Column, paths: Column*): Column = + Column.fn("variant_delete", (v +: path +: paths): _*) /** - * Locates the position of the first occurrence of the value in the given array as long. Returns - * null if either of the arguments are null. - * - * @param column - * The array to search. A column that evaluates to an array. - * @param value - * The value to locate. A column. - * @note - * The position is not zero based, but 1 based index. Returns 0 if value could not be found in - * array. + * Removes fields or array elements from a variant at the given JSONPath locations. Multiple + * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. * - * @group array_funcs - * @since 2.4.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the first JSONPath identifying a deletion target. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A string. Must be a constant. + * @param paths + * additional JSONPath strings, applied after `path` in order. A string. Must be a constant. + * @group variant_funcs + * @since 5.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a variant. */ - def array_position(column: Column, value: Any): Column = - Column.fn("array_position", column, lit(value)) + @scala.annotation.varargs + def variant_delete(v: Column, path: String, paths: String*): Column = + Column.fn("variant_delete", (v +: lit(path) +: paths.map(lit)): _*) /** - * Returns element of array at given index in value if column is array. Returns value for the - * given key in value if column is map. + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field (error if it already exists); an array path inserts at the index, shifting later + * elements right. Missing intermediate keys are created. Throws an error if a path segment hits + * a value of an incompatible type. Returns NULL if any argument is NULL. * - * @param column - * The array or map to extract from. A column that evaluates to an array or a map. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the insertion target. A valid path + * should start with `$` and is followed by one or more segments like `[123]`, `.name`, + * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a + * string. * @param value - * The 1-based index for arrays, or the key for maps. A column. - * @group collection_funcs - * @since 2.4.0 + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 * @return - * Returns a column of the element type of the input array, or the value type of the input - * map. + * Returns a column that evaluates to a variant. */ - def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) + def variant_insert(v: Column, path: Column, value: Column): Column = + Column.fn("variant_insert", v, path, value) /** - * (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will - * throw an error. If index < 0, accesses elements from the last to the first. The function - * always returns NULL if the index exceeds the length of the array. - * - * (map, key) - Returns value for given key. The function always returns NULL if the key is not - * contained in the map. + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field (error if it already exists); an array path inserts at the index, shifting later + * elements right. Missing intermediate keys are created. Throws an error if a path segment hits + * a value of an incompatible type. Returns NULL if any argument is NULL. * - * @param column - * The array or map to extract from. A column that evaluates to an array or a map. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the insertion target. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A string. Must be a constant. * @param value - * The 1-based index for arrays, or the key for maps. A column. - * @group collection_funcs - * @since 3.5.0 - * @return - * Returns a column of the element type of the input array, or the value type of the input - * map. - */ - def try_element_at(column: Column, value: Column): Column = - Column.fn("try_element_at", column, value) - - /** - * Returns element of array at given (0-based) index. If the index points outside of the array - * boundaries, then this function returns NULL. - * - * @param column - * The array to extract from. A column that evaluates to an array. - * @param index - * The 0-based index. A column that evaluates to an integral. - * @group array_funcs - * @since 3.4.0 + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 * @return - * Returns a column of the element type of the input array. + * Returns a column that evaluates to a variant. */ - def get(column: Column, index: Column): Column = Column.fn("get", column, index) + def variant_insert(v: Column, path: String, value: Column): Column = + Column.fn("variant_insert", v, lit(path), value) /** - * Sorts the input array in ascending order. Null elements will be placed at the end of the - * returned array. NaN is greater than any non-NaN elements for double/float type. - * - * The elements of the input array must be orderable. For example, when the array elements are - * structs, the default comparator compares the struct fields in schema order. Therefore, all - * fields in the struct must be orderable. If the default comparator does not support the input - * type, you can specify a custom comparator. + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field; an array path inserts at the index, shifting later elements right. Missing + * intermediate keys are created. Returns NULL if the field already exists or a path segment + * hits a value of an incompatible type, or if any argument is NULL. * - * @param e - * The array to sort. A column that evaluates to an array. - * @group collection_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the insertion target. A valid path + * should start with `$` and is followed by one or more segments like `[123]`, `.name`, + * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a + * string. + * @param value + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def array_sort(e: Column): Column = Column.fn("array_sort", e) + def try_variant_insert(v: Column, path: Column, value: Column): Column = + Column.fn("try_variant_insert", v, path, value) /** - * Sorts the input array based on the given comparator function. The comparator will take two - * arguments representing two elements of the array. It returns a negative integer, 0, or a - * positive integer as the first element is less than, equal to, or greater than the second - * element. If the comparator function returns null, the function will fail and raise an error. + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field; an array path inserts at the index, shifting later elements right. Missing + * intermediate keys are created. Returns NULL if the field already exists or a path segment + * hits a value of an incompatible type, or if any argument is NULL. * - * @param e - * The array to sort. A column that evaluates to an array. - * @param comparator - * A binary comparator function that returns a negative integer, 0, or a positive integer as - * the first element is less than, equal to, or greater than the second element. - * @group collection_funcs - * @since 3.4.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the insertion target. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A string. Must be a constant. + * @param value + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def array_sort(e: Column, comparator: (Column, Column) => Column): Column = - Column.fn("array_sort", e, createLambda(comparator)) + def try_variant_insert(v: Column, path: String, value: Column): Column = + Column.fn("try_variant_insert", v, lit(path), value) /** - * Remove all elements that equal to element from the given array. + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Throws an error if a path segment hits a value of an incompatible type. + * Returns NULL if any argument is NULL. * - * @param column - * The array to remove from. A column that evaluates to an array. - * @param element - * The element to remove. A column. - * @group array_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. + * @param value + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def array_remove(column: Column, element: Any): Column = - Column.fn("array_remove", column, lit(element)) + def variant_set(v: Column, path: Column, value: Column): Column = + Column.fn("variant_set", v, path, value) /** - * Remove all null elements from the given array. + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Throws an error if a path segment hits a value of an incompatible type. + * Returns NULL if any argument is NULL. * - * @param column - * The array to compact. A column that evaluates to an array. - * @group array_funcs - * @since 3.4.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. A string. Must be a constant. + * @param value + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def array_compact(column: Column): Column = Column.fn("array_compact", column) + def variant_set(v: Column, path: String, value: Column): Column = + Column.fn("variant_set", v, lit(path), value) /** - * Returns an array containing value as well as all elements from array. The new element is - * positioned at the beginning of the array. + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns + * NULL if any argument is NULL. * - * @param column - * The array to prepend to. A column that evaluates to an array. - * @param element - * The element to prepend. A column. - * @group array_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. A boolean. Must be a + * constant. + * @group variant_funcs + * @since 4.3.0 */ - def array_prepend(column: Column, element: Any): Column = - Column.fn("array_prepend", column, lit(element)) + def variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = + Column.fn("variant_set", v, path, value, lit(createIfMissing)) /** - * Removes duplicate values from the array. - * @param e - * The array to deduplicate. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to an array. + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns + * NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. A string. Must be a constant. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. A boolean. Must be a + * constant. + * @group variant_funcs + * @since 4.3.0 */ - def array_distinct(e: Column): Column = Column.fn("array_distinct", e) + def variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = + Column.fn("variant_set", v, lit(path), value, lit(createIfMissing)) /** - * Returns an array of the elements in the intersection of the given two arrays, without - * duplicates. + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if + * any argument is NULL. * - * @param col1 - * The first array. A column that evaluates to an array. - * @param col2 - * The second array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. + * @param value + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def array_intersect(col1: Column, col2: Column): Column = - Column.fn("array_intersect", col1, col2) + def try_variant_set(v: Column, path: Column, value: Column): Column = + Column.fn("try_variant_set", v, path, value) /** - * Adds an item into a given array at a specified position + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if + * any argument is NULL. * - * @param arr - * The array to insert into. A column that evaluates to an array. - * @param pos - * The 1-based position at which to insert (negative counts from the end). A column that - * evaluates to an integral. + * @param v + * a variant column. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. * @param value - * The value to insert. A column. - * @group array_funcs - * @since 3.4.0 - * @return - * Returns a column that evaluates to an array. + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def array_insert(arr: Column, pos: Column, value: Column): Column = - Column.fn("array_insert", arr, pos, value) + def try_variant_set(v: Column, path: String, value: Column): Column = + Column.fn("try_variant_set", v, lit(path), value) /** - * Returns an array of the elements in the union of the given two arrays, without duplicates. + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any + * argument is NULL. * - * @param col1 - * The first array. A column that evaluates to an array. - * @param col2 - * The second array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. + * @group variant_funcs + * @since 4.3.0 */ - def array_union(col1: Column, col2: Column): Column = - Column.fn("array_union", col1, col2) + def try_variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = + Column.fn("try_variant_set", v, path, value, lit(createIfMissing)) /** - * Returns an array of the elements in the first array but not in the second array, without - * duplicates. The order of elements in the result is not determined + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any + * argument is NULL. * - * @param col1 - * The first array. A column that evaluates to an array. - * @param col2 - * The second array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. + * @group variant_funcs + * @since 4.3.0 */ - def array_except(col1: Column, col2: Column): Column = - Column.fn("array_except", col1, col2) - - private def createLambda(f: Column => Column) = { - val x = internal.UnresolvedNamedLambdaVariable("x") - val function = f(Column(x)).node - Column(internal.LambdaFunction(function, Seq(x))) - } - - private def createLambda(f: (Column, Column) => Column) = { - val x = internal.UnresolvedNamedLambdaVariable("x") - val y = internal.UnresolvedNamedLambdaVariable("y") - val function = f(Column(x), Column(y)).node - Column(internal.LambdaFunction(function, Seq(x, y))) - } - - private def createLambda(f: (Column, Column, Column) => Column) = { - val x = internal.UnresolvedNamedLambdaVariable("x") - val y = internal.UnresolvedNamedLambdaVariable("y") - val z = internal.UnresolvedNamedLambdaVariable("z") - val function = f(Column(x), Column(y), Column(z)).node - Column(internal.LambdaFunction(function, Seq(x, y, z))) - } + def try_variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = + Column.fn("try_variant_set", v, lit(path), value, lit(createIfMissing)) /** - * Returns an array of elements after applying a transformation to each element in the input - * array. - * {{{ - * df.select(transform(col("i"), x => x + 1)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => transformed_col, the lambda function to transform the input column. + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of + * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. * - * @group collection_funcs - * @since 3.0.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the target array. A valid path should + * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. A column that evaluates to a string. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def transform(column: Column, f: Column => Column): Column = - Column.fn("transform", column, createLambda(f)) + def variant_array_append(v: Column, path: Column, value: Column): Column = + Column.fn("variant_array_append", v, path, value) /** - * Returns an array of elements after applying a transformation to each element in the input - * array. - * {{{ - * df.select(transform(col("i"), (x, i) => x + i)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * (col, index) => transformed_col, the lambda function to transform the input column given - * the index. Indices start at 0. + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of + * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. * - * @group collection_funcs - * @since 3.0.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the target array. A valid path should start with `$` and is + * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. A + * string. Must be a constant. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def transform(column: Column, f: (Column, Column) => Column): Column = - Column.fn("transform", column, createLambda(f)) + def variant_array_append(v: Column, path: String, value: Column): Column = + Column.fn("variant_array_append", v, lit(path), value) /** - * Returns whether a predicate holds for one or more elements in the array. - * {{{ - * df.select(exists(col("i"), _ % 2 === 0)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => predicate, the Boolean predicate to check the input column. + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an + * incompatible type, the target is not an array, or if any argument is NULL. * - * @group collection_funcs - * @since 3.0.0 - * @return - * Returns a column that evaluates to a boolean. + * @param v + * a variant column. + * @param path + * the column containing the JSONPath string identifying the target array. A valid path should + * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def exists(column: Column, f: Column => Column): Column = - Column.fn("exists", column, createLambda(f)) + def try_variant_array_append(v: Column, path: Column, value: Column): Column = + Column.fn("try_variant_array_append", v, path, value) /** - * Returns whether a predicate holds for every element in the array. - * {{{ - * df.select(forall(col("i"), x => x % 2 === 0)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => predicate, the Boolean predicate to check the input column. + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an + * incompatible type, the target is not an array, or if any argument is NULL. * - * @group collection_funcs - * @since 3.0.0 - * @return - * Returns a column that evaluates to a boolean. + * @param v + * a variant column. + * @param path + * the JSONPath identifying the target array. A valid path should start with `$` and is + * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def forall(column: Column, f: Column => Column): Column = - Column.fn("forall", column, createLambda(f)) + def try_variant_array_append(v: Column, path: String, value: Column): Column = + Column.fn("try_variant_array_append", v, lit(path), value) /** - * Returns an array of elements for which a predicate holds in a given array. - * {{{ - * df.select(filter(col("s"), x => x % 2 === 0)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => predicate, the Boolean predicate to filter the input column. + * Recursively removes object fields and array elements whose value is a variant null. Returns + * NULL if `v` is NULL. * - * @group collection_funcs - * @since 3.0.0 - * @return - * Returns a column that evaluates to an array. + * @param v + * a variant column. + * @group variant_funcs + * @since 4.3.0 */ - def filter(column: Column, f: Column => Column): Column = - Column.fn("filter", column, createLambda(f)) + def variant_strip_nulls(v: Column): Column = Column.fn("variant_strip_nulls", v) /** - * Returns an array of elements for which a predicate holds in a given array. - * {{{ - * df.select(filter(col("s"), (x, i) => i % 2 === 0)) - * }}} + * Recursively removes object fields and array elements whose value is a variant null, unless + * `includeArrays` is false, in which case null array elements are kept. Returns NULL if any + * argument is NULL. * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * (col, index) => predicate, the Boolean predicate to filter the input column given the - * index. Indices start at 0. + * @param v + * a variant column. + * @param includeArrays + * whether null elements are also removed from arrays. + * @group variant_funcs + * @since 4.3.0 + */ + def variant_strip_nulls(v: Column, includeArrays: Boolean): Column = + Column.fn("variant_strip_nulls", v, lit(includeArrays)) + + /** + * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. * - * @group collection_funcs - * @since 3.0.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the extraction path. A valid path should start with `$` and is followed by zero or more + * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the type specified by the `targetType` argument. */ - def filter(column: Column, f: (Column, Column) => Column): Column = - Column.fn("filter", column, createLambda(f)) + def variant_get(v: Column, path: String, targetType: String): Column = + Column.fn("variant_get", v, lit(path), lit(targetType)) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. The final state is converted into the final result by applying a finish - * function. - * {{{ - * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) - * }}} - * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value. - * @param finish - * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result. + * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. * - * @group collection_funcs - * @since 3.0.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the extraction path strings. A valid path string should start with + * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or + * `["name"]`. A column that evaluates to a string. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column of the type specified by the `targetType` argument. */ - def aggregate( - expr: Column, - initialValue: Column, - merge: (Column, Column) => Column, - finish: Column => Column): Column = - Column.fn("aggregate", expr, initialValue, createLambda(merge), createLambda(finish)) + def variant_get(v: Column, path: Column, targetType: String): Column = + Column.fn("variant_get", v, path, lit(targetType)) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. - * {{{ - * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x)) - * }}} + * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist or the cast fails.. * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value - * @group collection_funcs - * @since 3.0.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the extraction path. A valid path should start with `$` and is followed by zero or more + * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column of the type specified by the `targetType` argument. */ - def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = - aggregate(expr, initialValue, merge, c => c) + def try_variant_get(v: Column, path: String, targetType: String): Column = + Column.fn("try_variant_get", v, lit(path), lit(targetType)) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. The final state is converted into the final result by applying a finish - * function. - * {{{ - * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) - * }}} - * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value. - * @param finish - * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result. + * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist or the cast fails.. * - * @group collection_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the extraction path strings. A valid path string should start with + * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or + * `["name"]`. A column that evaluates to a string. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column of the type specified by the `targetType` argument. */ - def reduce( - expr: Column, - initialValue: Column, - merge: (Column, Column) => Column, - finish: Column => Column): Column = - Column.fn("reduce", expr, initialValue, createLambda(merge), createLambda(finish)) + def try_variant_get(v: Column, path: Column, targetType: String): Column = + Column.fn("try_variant_get", v, lit(path), lit(targetType)) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. - * {{{ - * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x)) - * }}} + * Returns schema in the SQL format of a variant. * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value - * @group collection_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column that evaluates to a string. */ - def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = - reduce(expr, initialValue, merge, c => c) + def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) /** - * Merge two given arrays, element-wise, into a single array using a function. If one array is - * shorter, nulls are appended at the end to match the length of the longer array, before - * applying the function. - * {{{ - * df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y)) - * }}} - * - * @param left - * the left input array column. A column that evaluates to an array. - * @param right - * the right input array column. A column that evaluates to an array. - * @param f - * (lCol, rCol) => col, the lambda function to merge two input columns into one column. + * Returns the merged schema in the SQL format of a variant column. * - * @group collection_funcs - * @since 3.0.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = - Column.fn("zip_with", left, right, createLambda(f)) + def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // XML Functions + ////////////////////////////////////////////////////////////////////////////////////////////// + // scalastyle:off line.size.limit /** - * Applies a function to every key-value pair in a map and returns a map with the results of - * those applications as the new keys for the pairs. - * {{{ - * df.select(transform_keys(col("i"), (k, v) => k + v)) - * }}} - * - * @param expr - * the input map column. A column that evaluates to a map. - * @param f - * (key, value) => new_key, the lambda function to transform the key of input map column + * Parses a column containing a XML string into the data type corresponding to the specified + * schema. Returns `null`, in the case of an unparseable string. * - * @group collection_funcs - * @since 3.0.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the XML is parsed. accepts the same options and the XML data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a struct. */ - def transform_keys(expr: Column, f: (Column, Column) => Column): Column = - Column.fn("transform_keys", expr, createLambda(f)) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = + from_xml(e, lit(schema.sql), options.asScala.iterator) + // scalastyle:off line.size.limit /** - * Applies a function to every key-value pair in a map and returns a map with the results of - * those applications as the new values for the pairs. - * {{{ - * df.select(transform_values(col("i"), (k, v) => k + v)) - * }}} - * - * @param expr - * the input map column. A column that evaluates to a map. - * @param f - * (key, value) => new_value, the lambda function to transform the value of input map column + * (Java-specific) Parses a column containing a XML string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @group collection_funcs - * @since 3.0.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. + * @param options + * options to control how the XML is parsed. accepts the same options and the xml data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a struct. */ - def transform_values(expr: Column, f: (Column, Column) => Column): Column = - Column.fn("transform_values", expr, createLambda(f)) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { + from_xml(e, lit(schema), options) + } + // scalastyle:off line.size.limit /** - * Returns a map whose key-value pairs satisfy a predicate. - * {{{ - * df.select(map_filter(col("m"), (k, v) => k * 10 === v)) - * }}} - * - * @param expr - * the input map column. A column that evaluates to a map. - * @param f - * (key, value) => predicate, the Boolean predicate to filter the input map column + * (Java-specific) Parses a column containing a XML string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @group collection_funcs - * @since 3.0.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A column that evaluates to a string. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a struct. */ - def map_filter(expr: Column, f: (Column, Column) => Column): Column = - Column.fn("map_filter", expr, createLambda(f)) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: Column): Column = { + from_xml(e, schema, Iterator.empty) + } + // scalastyle:off line.size.limit /** - * Merge two given maps, key-wise into a single map using a function. - * {{{ - * df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)) - * }}} - * - * @param left - * the left input map column. A column that evaluates to a map. - * @param right - * the right input map column. A column that evaluates to a map. - * @param f - * (key, value1, value2) => new_value, the lambda function to merge the map values + * (Java-specific) Parses a column containing a XML string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @group collection_funcs - * @since 3.0.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A column that evaluates to a string. + * @param options + * options to control how the XML is parsed. accepts the same options and the XML data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a struct. */ - def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = - Column.fn("map_zip_with", left, right, createLambda(f)) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = + from_xml(e, schema, options.asScala.iterator) /** - * Creates a new row for each element in the given array or map column. Uses the default column - * name `col` for elements in the array and `key` and `value` for elements in the map unless - * specified otherwise. + * Parses a column containing a XML string into the data type corresponding to the specified + * schema. Returns `null`, in the case of an unparseable string. * * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 1.3.0 + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a + * constant. + * + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column of the element type of the input array, or the key and value columns of - * the input map. + * Returns a column that evaluates to a struct. */ - def explode(e: Column): Column = Column.fn("explode", e) + def from_xml(e: Column, schema: StructType): Column = + from_xml(e, schema, Map.empty[String, String].asJava) + + private def from_xml(e: Column, schema: Column, options: Iterator[(String, String)]): Column = { + Column.fnWithOptions("from_xml", options, e, schema) + } /** - * Creates a new row for each element in the given array or map column. Uses the default column - * name `col` for elements in the array and `key` and `value` for elements in the map unless - * specified otherwise. Unlike explode, if the array/map is null or empty then null is produced. + * Parses a XML string and infers its schema in DDL format. * - * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 2.2.0 + * @param xml + * a XML string. A string. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column of the element type of the input array, or the key and value columns of - * the input map. + * Returns a column that evaluates to a string. */ - def explode_outer(e: Column): Column = Column.fn("explode_outer", e) + def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) /** - * Creates a new row for each element with position in the given array or map column. Uses the - * default column name `pos` for position, and `col` for elements in the array and `key` and - * `value` for elements in the map unless specified otherwise. + * Parses a XML string and infers its schema in DDL format. * - * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 2.1.0 + * @param xml + * a foldable string column containing a XML string. A column that evaluates to a string. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns the position column and a column of the element type of the input array, or the - * position column and the key and value columns of the input map. + * Returns a column that evaluates to a string. */ - def posexplode(e: Column): Column = Column.fn("posexplode", e) + def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) + + // scalastyle:off line.size.limit /** - * Creates a new row for each element with position in the given array or map column. Uses the - * default column name `pos` for position, and `col` for elements in the array and `key` and - * `value` for elements in the map unless specified otherwise. Unlike posexplode, if the - * array/map is null or empty then the row (null, null) is produced. + * Parses a XML string and infers its schema in DDL format using options. * - * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 2.2.0 + * @param xml + * a foldable string column containing XML data. A column that evaluates to a string. + * @param options + * options to control how the xml is parsed. accepts the same options and the XML data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * Returns the position column and a column of the element type of the input array, or the - * position column and the key and value columns of the input map. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. + * @group xml_funcs + * @since 4.0.0 */ - def posexplode_outer(e: Column): Column = Column.fn("posexplode_outer", e) + // scalastyle:on line.size.limit + def schema_of_xml(xml: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("schema_of_xml", options.asScala.iterator, xml) + + // scalastyle:off line.size.limit /** - * Creates a new row for each element in the given array of structs. + * (Java-specific) Converts a column containing a `StructType` into a XML string with the + * specified schema. Throws an exception, in the case of an unsupported type. * * @param e - * the target column to explode. A column that evaluates to an array of structs. - * @group generator_funcs - * @since 3.4.0 + * a column containing a struct. A column that evaluates to a string. + * @param options + * options to control how the struct column is converted into a XML string. It accepts the + * same options as the XML data source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - def inline(e: Column): Column = Column.fn("inline", e) + // scalastyle:on line.size.limit + def to_xml(e: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("to_xml", options.asScala.iterator, e) /** - * Creates a new row for each element in the given array of structs. Unlike inline, if the array - * is null or empty then null is produced for each nested column. + * Converts a column containing a `StructType` into a XML string with the specified schema. + * Throws an exception, in the case of an unsupported type. * * @param e - * the target column to explode. A column that evaluates to an array of structs. - * @group generator_funcs - * @since 3.4.0 + * a column containing a struct. A column that evaluates to a string. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - def inline_outer(e: Column): Column = Column.fn("inline_outer", e) + def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) /** - * Extracts json object from a json string based on json path specified, and returns json string - * of the extracted json object. It will return null if the input json string is invalid. + * Returns a string array of values within the nodes of xml that match the XPath expression. * - * @param e - * the JSON string column. A column that evaluates to a string. + * @param xml + * the XML column to evaluate. A column that evaluates to a string. * @param path - * the JSON path to extract. A column that evaluates to a string. Must be a constant. - * @group json_funcs - * @since 1.6.0 + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def get_json_object(e: Column, path: String): Column = - Column.fn("get_json_object", e, lit(path)) + def xpath(xml: Column, path: Column): Column = + Column.fn("xpath", xml, path) /** - * Creates a new row for a json column according to the given field names. + * Returns true if the XPath expression evaluates to true, or if a matching node is found. * - * @param json - * the JSON string column. A column that evaluates to a string. - * @param fields - * the field names to extract. A column that evaluates to a string. Must be a constant. - * @group json_funcs - * @since 1.6.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a boolean. */ - @scala.annotation.varargs - def json_tuple(json: Column, fields: String*): Column = { - require(fields.nonEmpty, "at least 1 field name should be given.") - Column.fn("json_tuple", json +: fields.map(lit): _*) - } + def xpath_boolean(xml: Column, path: Column): Column = + Column.fn("xpath_boolean", xml, path) - // scalastyle:off line.size.limit /** - * (Scala-specific) Parses a column containing a JSON string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. Accepts the same options as the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Returns a double value, the value zero if no match is found, or NaN if a match is found but + * the value is non-numeric. * - * @group json_funcs - * @since 2.1.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = - from_json(e, schema.asInstanceOf[DataType], options) + def xpath_double(xml: Column, path: Column): Column = + Column.fn("xpath_double", xml, path) - // scalastyle:off line.size.limit /** - * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Returns a double value, the value zero if no match is found, or NaN if a match is found but + * the value is non-numeric. * - * @group json_funcs - * @since 2.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a double. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { - from_json(e, lit(schema.sql), options.iterator) - } + def xpath_number(xml: Column, path: Column): Column = + Column.fn("xpath_number", xml, path) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a JSON string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Returns a float value, the value zero if no match is found, or NaN if a match is found but + * the value is non-numeric. * - * @group json_funcs - * @since 2.1.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a float. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = - from_json(e, schema, options.asScala.toMap) + def xpath_float(xml: Column, path: Column): Column = + Column.fn("xpath_float", xml, path) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Returns an integer value, or the value zero if no match is found, or a match is found but the + * value is non-numeric. * - * @group json_funcs - * @since 2.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to an integer. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { - from_json(e, schema, options.asScala.toMap) - } + def xpath_int(xml: Column, path: Column): Column = + Column.fn("xpath_int", xml, path) /** - * Parses a column containing a JSON string into a `StructType` with the specified schema. - * Returns `null`, in the case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. + * Returns a long integer value, or the value zero if no match is found, or a match is found but + * the value is non-numeric. * - * @group json_funcs - * @since 2.1.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a long. */ - def from_json(e: Column, schema: StructType): Column = - from_json(e, schema, Map.empty[String, String]) + def xpath_long(xml: Column, path: Column): Column = + Column.fn("xpath_long", xml, path) /** - * Parses a column containing a JSON string into a `MapType` with `StringType` as keys type, - * `StructType` or `ArrayType` with the specified schema. Returns `null`, in the case of an - * unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. + * Returns a short integer value, or the value zero if no match is found, or a match is found + * but the value is non-numeric. * - * @group json_funcs - * @since 2.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a short. */ - def from_json(e: Column, schema: DataType): Column = - from_json(e, schema, Map.empty[String, String]) + def xpath_short(xml: Column, path: Column): Column = + Column.fn("xpath_short", xml, path) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Returns the text contents of the first xml node that matches the XPath expression. * - * @group json_funcs - * @since 2.1.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a string. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { - from_json(e, schema, options.asScala.toMap) - } + def xpath_string(xml: Column, path: Column): Column = + Column.fn("xpath_string", xml, path) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // URL Functions + ////////////////////////////////////////////////////////////////////////////////////////////// - // scalastyle:off line.size.limit /** - * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Extracts a part from a URL. * - * @group json_funcs - * @since 2.3.0 + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. + * @group url_funcs + * @since 4.0.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a string. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: String, options: Map[String, String]): Column = { - from_json(e, lit(schema), options.asJava) - } + def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = + Column.fn("try_parse_url", url, partToExtract, key) /** - * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns - * `null`, in the case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A column that evaluates to a string. + * Extracts a part from a URL. * - * @group json_funcs - * @since 2.4.0 + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @group url_funcs + * @since 4.0.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a string. */ - def from_json(e: Column, schema: Column): Column = { - from_json(e, schema, Map.empty[String, String].asJava) - } + def try_parse_url(url: Column, partToExtract: Column): Column = + Column.fn("try_parse_url", url, partToExtract) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns - * `null`, in the case of an unparseable string. - * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A column that evaluates to a string. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Extracts a part from a URL. * - * @group json_funcs - * @since 2.4.0 + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. + * @group url_funcs + * @since 3.5.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a string. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { - from_json(e, schema, options.asScala.iterator) - } - - private def from_json( - e: Column, - schema: Column, - options: Iterator[(String, String)]): Column = { - Column.fnWithOptions("from_json", options, e, schema) - } + def parse_url(url: Column, partToExtract: Column, key: Column): Column = + Column.fn("parse_url", url, partToExtract, key) /** - * Parses a JSON string and constructs a Variant value. Returns null if the input string is not - * a valid JSON value. - * - * @param json - * a string column that contains JSON data. A column that evaluates to a string. + * Extracts a part from a URL. * - * @group variant_funcs - * @since 4.0.0 + * @param url + * A column representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @group url_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) + def parse_url(url: Column, partToExtract: Column): Column = + Column.fn("parse_url", url, partToExtract) /** - * Parses a JSON string and constructs a Variant value. + * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding + * scheme. * - * @param json - * a string column that contains JSON data. A column that evaluates to a string. - * @group variant_funcs - * @since 4.0.0 + * @param str + * A URL-encoded string. A column that evaluates to a string. + * @group url_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - def parse_json(json: Column): Column = Column.fn("parse_json", json) + def url_decode(str: Column): Column = Column.fn("url_decode", str) /** - * Converts a column containing nested inputs (array/map/struct) into a variants where maps and - * structs are converted to variant objects which are unordered unlike SQL structs. Input maps - * can only have string keys. + * This is a special version of `url_decode` that performs the same operation, but returns a + * NULL value instead of raising an error if the decoding cannot be performed. * - * @param col - * a column with a nested schema or column name. A column that evaluates to a struct, array, - * map, or variant. - * @group variant_funcs + * @param str + * A URL-encoded string. A column that evaluates to a string. + * @group url_funcs * @since 4.0.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) + def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) /** - * Creates a variant object from the given arrays of keys and values. The keys must be non-null - * strings and the two arrays must have the same length. + * Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding + * scheme. * - * @param keys - * a column that evaluates to an array of string keys. - * @param values - * a column that evaluates to an array of values. - * @group variant_funcs - * @since 4.4.0 + * @param str + * A string to encode. A column that evaluates to a string. + * @group url_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - def variant_from_arrays(keys: Column, values: Column): Column = - Column.fn("variant_from_arrays", keys, values) + def url_encode(str: Column): Column = Column.fn("url_encode", str) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Misc Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Creates a variant object from an array of key/value struct entries. The keys must be non-null - * strings. + * Creates a string column for the file name of the current Spark task. * - * @param entries - * a column that evaluates to an array of key/value structs. - * @group variant_funcs - * @since 4.4.0 + * @group misc_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - def variant_from_entries(entries: Column): Column = - Column.fn("variant_from_entries", entries) + def input_file_name(): Column = Column.fn("input_file_name") /** - * Check if a variant value is a variant null. Returns true if and only if the input is a - * variant null and false otherwise (including in the case of SQL NULL). + * A column expression that generates monotonically increasing 64-bit integers. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.0.0 + * The generated ID is guaranteed to be monotonically increasing and unique, but not + * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the + * record number within each partition in the lower 33 bits. The assumption is that the data + * frame has less than 1 billion partitions, and each partition has less than 8 billion records. + * + * As an example, consider a `DataFrame` with two partitions, each with 3 records. This + * expression would return the following IDs: + * + * {{{ + * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. + * }}} + * + * @group misc_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a long. */ - def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) + @deprecated("Use monotonically_increasing_id()", "2.0.0") + def monotonicallyIncreasingId(): Column = monotonically_increasing_id() /** - * Check if a variant value is valid. Returns true if the variant is valid, false if it is - * malformed, and NULL if the input is NULL. + * A column expression that generates monotonically increasing 64-bit integers. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.2.0 + * The generated ID is guaranteed to be monotonically increasing and unique, but not + * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the + * record number within each partition in the lower 33 bits. The assumption is that the data + * frame has less than 1 billion partitions, and each partition has less than 8 billion records. + * + * As an example, consider a `DataFrame` with two partitions, each with 3 records. This + * expression would return the following IDs: + * + * {{{ + * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. + * }}} + * + * @group misc_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a long. */ - def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) + def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") /** - * Removes fields or array elements from a variant at the given JSONPath locations. Multiple - * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. + * Partition ID. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the first JSONPath string. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A column that evaluates to a string. - * @param paths - * additional JSONPath arguments, applied after `path` in order. A column that evaluates to a - * string. - * @group variant_funcs - * @since 5.0.0 + * @note + * This is non-deterministic because it depends on data partitioning and task scheduling. + * + * @group misc_funcs + * @since 1.6.0 + * @return + * Returns a column that evaluates to an integer. + */ + def spark_partition_id(): Column = Column.fn("spark_partition_id") + + /** + * Returns the current catalog. + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def variant_delete(v: Column, path: Column, paths: Column*): Column = - Column.fn("variant_delete", (v +: path +: paths): _*) + def current_catalog(): Column = Column.fn("current_catalog") /** - * Removes fields or array elements from a variant at the given JSONPath locations. Multiple - * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. + * Returns the current database. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the first JSONPath identifying a deletion target. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A string. Must be a constant. - * @param paths - * additional JSONPath strings, applied after `path` in order. A string. Must be a constant. - * @group variant_funcs - * @since 5.0.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def variant_delete(v: Column, path: String, paths: String*): Column = - Column.fn("variant_delete", (v +: lit(path) +: paths.map(lit)): _*) + def current_database(): Column = Column.fn("current_database") /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field (error if it already exists); an array path inserts at the index, shifting later - * elements right. Missing intermediate keys are created. Throws an error if a path segment hits - * a value of an incompatible type. Returns NULL if any argument is NULL. + * Returns the current schema. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the insertion target. A valid path - * should start with `$` and is followed by one or more segments like `[123]`, `.name`, - * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a - * string. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - def variant_insert(v: Column, path: Column, value: Column): Column = - Column.fn("variant_insert", v, path, value) + def current_schema(): Column = Column.fn("current_schema") /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field (error if it already exists); an array path inserts at the index, shifting later - * elements right. Missing intermediate keys are created. Throws an error if a path segment hits - * a value of an incompatible type. Returns NULL if any argument is NULL. + * Returns the current SQL path as a comma-separated list of qualified schema names. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the insertion target. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A string. Must be a constant. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @group misc_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a string. */ - def variant_insert(v: Column, path: String, value: Column): Column = - Column.fn("variant_insert", v, lit(path), value) + def current_path(): Column = Column.fn("current_path") /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field; an array path inserts at the index, shifting later elements right. Missing - * intermediate keys are created. Returns NULL if the field already exists or a path segment - * hits a value of an incompatible type, or if any argument is NULL. + * Returns the user name of current execution context. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the insertion target. A valid path - * should start with `$` and is followed by one or more segments like `[123]`, `.name`, - * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a - * string. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ - def try_variant_insert(v: Column, path: Column, value: Column): Column = - Column.fn("try_variant_insert", v, path, value) + def current_user(): Column = Column.fn("current_user") /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field; an array path inserts at the index, shifting later elements right. Missing - * intermediate keys are created. Returns NULL if the field already exists or a path segment - * hits a value of an incompatible type, or if any argument is NULL. + * Returns null if the condition is true, and throws an exception otherwise. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the insertion target. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A string. Must be a constant. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param c + * The condition to check. A column that evaluates to a boolean. + * @group misc_funcs + * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ - def try_variant_insert(v: Column, path: String, value: Column): Column = - Column.fn("try_variant_insert", v, lit(path), value) + def assert_true(c: Column): Column = Column.fn("assert_true", c) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Throws an error if a path segment hits a value of an incompatible type. - * Returns NULL if any argument is NULL. + * Returns null if the condition is true; throws an exception with the error message otherwise. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. - * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param c + * The condition to check. A column that evaluates to a boolean. + * @param e + * The error message to throw. A column that evaluates to a string. + * @group misc_funcs + * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ - def variant_set(v: Column, path: Column, value: Column): Column = - Column.fn("variant_set", v, path, value) + def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Throws an error if a path segment hits a value of an incompatible type. - * Returns NULL if any argument is NULL. + * Throws an exception with the provided error message. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. A string. Must be a constant. - * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param c + * The error message to throw. A column that evaluates to a string. + * @group misc_funcs + * @since 3.1.0 + * @return + * Returns a column that always evaluates to NULL. */ - def variant_set(v: Column, path: String, value: Column): Column = - Column.fn("variant_set", v, lit(path), value) + def raise_error(c: Column): Column = Column.fn("raise_error", c) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns - * NULL if any argument is NULL. + * Returns the user name of current execution context. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. A boolean. Must be a - * constant. - * @group variant_funcs - * @since 4.3.0 + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ - def variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = - Column.fn("variant_set", v, path, value, lit(createIfMissing)) + def user(): Column = Column.fn("user") /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns - * NULL if any argument is NULL. + * Returns the user name of current execution context. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. A string. Must be a constant. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. A boolean. Must be a - * constant. - * @group variant_funcs - * @since 4.3.0 + * @group misc_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ - def variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = - Column.fn("variant_set", v, lit(path), value, lit(createIfMissing)) + def session_user(): Column = Column.fn("session_user") /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if - * any argument is NULL. + * Returns an universally unique identifier (UUID) string. The value is returned as a canonical + * UUID 36-character string. * - * @param v - * a variant column. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a string. */ - def try_variant_set(v: Column, path: Column, value: Column): Column = - Column.fn("try_variant_set", v, path, value) + def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if - * any argument is NULL. + * Returns an universally unique identifier (UUID) string. The value is returned as a canonical + * UUID 36-character string. * - * @param v - * a variant column. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param seed + * The random number seed to use. A column that evaluates to an integral. Must be a constant. + * @group misc_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a string. */ - def try_variant_set(v: Column, path: String, value: Column): Column = - Column.fn("try_variant_set", v, lit(path), value) + def uuid(seed: Column): Column = Column.fn("uuid", seed) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any - * argument is NULL. + * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the + * given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with `hex` or + * `base64` for a textual value. * - * @param v - * a variant column. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. - * @group variant_funcs + * @param key + * The secret key, as a binary value. + * @param message + * The message to authenticate, as a binary value. + * @param algorithm + * The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. + * + * @group misc_funcs * @since 4.3.0 */ - def try_variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = - Column.fn("try_variant_set", v, path, value, lit(createIfMissing)) + def hmac(key: Column, message: Column, algorithm: Column): Column = + Column.fn("hmac", key, message, algorithm) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any - * argument is NULL. + * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and + * SHA-256. The result is returned as raw MAC bytes; wrap it with `hex` or `base64` for a + * textual value. To use a different algorithm, call the three-argument overload. * - * @param v - * a variant column. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. - * @group variant_funcs + * @param key + * The secret key, as a binary value. + * @param message + * The message to authenticate, as a binary value. + * + * @group misc_funcs * @since 4.3.0 */ - def try_variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = - Column.fn("try_variant_set", v, lit(path), value, lit(createIfMissing)) + def hmac(key: Column, message: Column): Column = + Column.fn("hmac", key, message) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of - * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + * Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. + * Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, + * `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization + * vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 + * bytes for GCM. If not provided, a random vector will be generated and prepended to the + * output. Optional additional authenticated data (AAD) is only supported for GCM. If provided + * for encryption, the identical AAD value must be provided for decryption. The default mode is + * GCM. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the target array. A valid path should - * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. A column that evaluates to a string. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param iv + * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. + * + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def variant_array_append(v: Column, path: Column, value: Column): Column = - Column.fn("variant_array_append", v, path, value) + def aes_encrypt( + input: Column, + key: Column, + mode: Column, + padding: Column, + iv: Column, + aad: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv, aad) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of - * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + * Returns an encrypted value of `input`. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the target array. A valid path should start with `$` and is - * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. A - * string. Must be a constant. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param iv + * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def variant_array_append(v: Column, path: String, value: Column): Column = - Column.fn("variant_array_append", v, lit(path), value) + def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = + Column.fn("aes_encrypt", input, key, mode, padding, iv) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an - * incompatible type, the target is not an array, or if any argument is NULL. + * Returns an encrypted value of `input`. * - * @param v - * a variant column. - * @param path - * the column containing the JSONPath string identifying the target array. A valid path should - * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def try_variant_array_append(v: Column, path: Column, value: Column): Column = - Column.fn("try_variant_array_append", v, path, value) + def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = + Column.fn("aes_encrypt", input, key, mode, padding) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an - * incompatible type, the target is not an array, or if any argument is NULL. + * Returns an encrypted value of `input`. * - * @param v - * a variant column. - * @param path - * the JSONPath identifying the target array. A valid path should start with `$` and is - * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def try_variant_array_append(v: Column, path: String, value: Column): Column = - Column.fn("try_variant_array_append", v, lit(path), value) + def aes_encrypt(input: Column, key: Column, mode: Column): Column = + Column.fn("aes_encrypt", input, key, mode) /** - * Recursively removes object fields and array elements whose value is a variant null. Returns - * NULL if `v` is NULL. + * Returns an encrypted value of `input`. * - * @param v - * a variant column. - * @group variant_funcs - * @since 4.3.0 + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def variant_strip_nulls(v: Column): Column = Column.fn("variant_strip_nulls", v) + def aes_encrypt(input: Column, key: Column): Column = + Column.fn("aes_encrypt", input, key) /** - * Recursively removes object fields and array elements whose value is a variant null, unless - * `includeArrays` is false, in which case null array elements are kept. Returns NULL if any - * argument is NULL. + * Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, + * 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', + * 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is + * only supported for GCM. If provided for encryption, the identical AAD value must be provided + * for decryption. The default mode is GCM. * - * @param v - * a variant column. - * @param includeArrays - * whether null elements are also removed from arrays. - * @group variant_funcs - * @since 4.3.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. + * + * @group misc_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def variant_strip_nulls(v: Column, includeArrays: Boolean): Column = - Column.fn("variant_strip_nulls", v, lit(includeArrays)) + def aes_decrypt( + input: Column, + key: Column, + mode: Column, + padding: Column, + aad: Column): Column = + Column.fn("aes_decrypt", input, key, mode, padding, aad) /** - * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. + * Returns a decrypted value of `input`. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the extraction path. A valid path should start with `$` and is followed by zero or more - * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column that evaluates to a binary. */ - def variant_get(v: Column, path: String, targetType: String): Column = - Column.fn("variant_get", v, lit(path), lit(targetType)) + def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = + Column.fn("aes_decrypt", input, key, mode, padding) /** - * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. + * Returns a decrypted value of `input`. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the extraction path strings. A valid path string should start with - * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or - * `["name"]`. A column that evaluates to a string. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column that evaluates to a binary. */ - def variant_get(v: Column, path: Column, targetType: String): Column = - Column.fn("variant_get", v, path, lit(targetType)) + def aes_decrypt(input: Column, key: Column, mode: Column): Column = + Column.fn("aes_decrypt", input, key, mode) /** - * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist or the cast fails.. + * Returns a decrypted value of `input`. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the extraction path. A valid path should start with `$` and is followed by zero or more - * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @see + * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column that evaluates to a binary. */ - def try_variant_get(v: Column, path: String, targetType: String): Column = - Column.fn("try_variant_get", v, lit(path), lit(targetType)) + def aes_decrypt(input: Column, key: Column): Column = + Column.fn("aes_decrypt", input, key) /** - * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist or the cast fails.. + * This is a special version of `aes_decrypt` that performs the same operation, but returns a + * NULL value instead of raising an error if the decryption cannot be performed. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the extraction path strings. A valid path string should start with - * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or - * `["name"]`. A column that evaluates to a string. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column that evaluates to a binary. */ - def try_variant_get(v: Column, path: Column, targetType: String): Column = - Column.fn("try_variant_get", v, lit(path), lit(targetType)) + def try_aes_decrypt( + input: Column, + key: Column, + mode: Column, + padding: Column, + aad: Column): Column = + Column.fn("try_aes_decrypt", input, key, mode, padding, aad) /** - * Returns schema in the SQL format of a variant. + * Returns a decrypted value of `input`. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) + def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = + Column.fn("try_aes_decrypt", input, key, mode, padding) /** - * Returns the merged schema in the SQL format of a variant column. + * Returns a decrypted value of `input`. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) + def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = + Column.fn("try_aes_decrypt", input, key, mode) /** - * Parses a JSON string and infers its schema in DDL format. + * Returns a decrypted value of `input`. * - * @param json - * a JSON string. A string. Must be a constant. + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @see + * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` * - * @group json_funcs - * @since 2.4.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def schema_of_json(json: String): Column = schema_of_json(lit(json)) + def try_aes_decrypt(input: Column, key: Column): Column = + Column.fn("try_aes_decrypt", input, key) /** - * Parses a JSON string and infers its schema in DDL format. - * - * @param json - * a foldable string column containing a JSON string. A column that evaluates to a string. + * Returns the length of the block being read, or -1 if not available. * - * @group json_funcs - * @since 2.4.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) + def input_file_block_length(): Column = Column.fn("input_file_block_length") - // scalastyle:off line.size.limit /** - * Parses a JSON string and infers its schema in DDL format using options. + * Returns the start offset of the block being read, or -1 if not available. * - * @param json - * a foldable string column containing JSON data. A column that evaluates to a string. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * @group misc_funcs + * @since 3.5.0 * @return - * a column with string literal containing schema in DDL format. Returns a column that - * evaluates to a string. - * - * @group json_funcs - * @since 3.0.0 + * Returns a column that evaluates to a long. */ - // scalastyle:on line.size.limit - def schema_of_json(json: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("schema_of_json", options.asScala.iterator, json) + def input_file_block_start(): Column = Column.fn("input_file_block_start") /** - * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of any - * other valid JSON string, `NULL` or an invalid JSON. + * Calls a method with reflection. * - * @param e - * the JSON array string column. A column that evaluates to a string. - * @group json_funcs + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a string. */ - def json_array_length(e: Column): Column = Column.fn("json_array_length", e) + @scala.annotation.varargs + def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) /** - * Returns all the keys of the outermost JSON object as an array. If a valid JSON object is - * given, all the keys of the outermost object will be returned as an array. If it is any other - * valid JSON string, an invalid JSON string or an empty string, the function returns null. + * Calls a method with reflection. * - * @param e - * the JSON object string column. A column that evaluates to a string. - * @group json_funcs + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) + @scala.annotation.varargs + def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) /** - * Returns the type of the outermost JSON value as a string: one of 'object', 'array', 'string', - * 'number', 'boolean', or 'null'. Returns null for invalid or empty input. + * This is a special version of `reflect` that performs the same operation, but returns a NULL + * value instead of raising an error if the invoke method thrown exception. * - * @param e - * the JSON string column. A column that evaluates to a string. - * @group json_funcs - * @since 4.4.0 + * @group misc_funcs + * @since 4.0.0 * @return * Returns a column that evaluates to a string. */ - def json_typeof(e: Column): Column = Column.fn("json_typeof", e) + @scala.annotation.varargs + def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) - // scalastyle:off line.size.limit /** - * (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into - * a JSON string with the specified schema. Throws an exception, in the case of an unsupported - * type. - * - * @param e - * a column containing a struct, an array, a map, or a variant. A column that evaluates to a - * struct, array, map, or variant. - * @param options - * options to control how the struct column is converted into a json string. accepts the same - * options and the json data source. See Data - * Source Option in the version you use. Additionally the function supports the `pretty` - * option which enables pretty JSON generation. A map of string options. Must be a constant. + * Returns the Spark version. The string contains 2 fields, the first being a release version + * and the second being a git revision. * - * @group json_funcs - * @since 2.1.0 + * @group misc_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to a string. */ - // scalastyle:on line.size.limit - def to_json(e: Column, options: Map[String, String]): Column = - Column.fnWithOptions("to_json", options.iterator, e) + def version(): Column = Column.fn("version") - // scalastyle:off line.size.limit /** - * (Java-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into - * a JSON string with the specified schema. Throws an exception, in the case of an unsupported - * type. - * - * @param e - * a column containing a struct, an array, a map, or a variant. A column that evaluates to a - * struct, array, map, or variant. - * @param options - * options to control how the struct column is converted into a json string. accepts the same - * options and the json data source. See Data - * Source Option in the version you use. Additionally the function supports the `pretty` - * option which enables pretty JSON generation. A map of string options. Must be a constant. + * Return DDL-formatted type string for the data type of the input. * - * @group json_funcs - * @since 2.1.0 + * @param col + * The value whose data type is returned. A column of any type. + * @group misc_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to a string. */ - // scalastyle:on line.size.limit - def to_json(e: Column, options: java.util.Map[String, String]): Column = - to_json(e, options.asScala.toMap) + def typeof(col: Column): Column = Column.fn("typeof", col) /** - * Converts a column containing a `StructType`, `ArrayType` or a `MapType` into a JSON string - * with the specified schema. Throws an exception, in the case of an unsupported type. - * - * @param e - * a column containing a struct, an array, a map, or a variant. A column that evaluates to a - * struct, array, map, or variant. + * Returns the bit position for the given input column. * - * @group json_funcs - * @since 2.1.0 + * @param col + * The input column. A column that evaluates to an integral. + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def to_json(e: Column): Column = - to_json(e, Map.empty[String, String]) + def bitmap_bit_position(col: Column): Column = + Column.fn("bitmap_bit_position", col) /** - * Masks the given string value. The function replaces characters with 'X' or 'x', and numbers - * with 'n'. This can be useful for creating copies of tables with sensitive information - * removed. - * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. + * Returns the bucket number for the given input column. * - * @group string_funcs + * @param col + * The input column. A column that evaluates to an integral. + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def mask(input: Column): Column = Column.fn("mask", input) + def bitmap_bucket_number(col: Column): Column = + Column.fn("bitmap_bucket_number", col) /** - * Masks the given string value. The function replaces upper-case characters with specific - * character, lower-case characters with 'x', and numbers with 'n'. This can be useful for - * creating copies of tables with sensitive information removed. - * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. + * Returns the number of set bits in the input bitmap. * - * @group string_funcs + * @param col + * The input bitmap. A column that evaluates to a binary. + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def mask(input: Column, upperChar: Column): Column = - Column.fn("mask", input, upperChar) + def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) /** - * Masks the given string value. The function replaces upper-case and lower-case characters with - * the characters specified respectively, and numbers with 'n'. This can be useful for creating - * copies of tables with sensitive information removed. - * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param lowerChar - * character to replace lower-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. + * Returns a bitmap that is the bitwise AND of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * - * @group string_funcs - * @since 3.5.0 + * @param left + * A column that evaluates to a binary bitmap. + * @param right + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary bitmap. */ - def mask(input: Column, upperChar: Column, lowerChar: Column): Column = - Column.fn("mask", input, upperChar, lowerChar) + def bitmap_and(left: Column, right: Column): Column = Column.fn("bitmap_and", left, right) /** - * Masks the given string value. The function replaces upper-case, lower-case characters and - * numbers with the characters specified respectively. This can be useful for creating copies of - * tables with sensitive information removed. + * Returns a bitmap that is the bitwise OR of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param lowerChar - * character to replace lower-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param digitChar - * character to replace digit characters with. Specify NULL to retain original character. A - * column that evaluates to a string. + * @param left + * A column that evaluates to a binary bitmap. + * @param right + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 + * @return + * Returns a column that evaluates to a binary bitmap. + */ + def bitmap_or(left: Column, right: Column): Column = Column.fn("bitmap_or", left, right) + + /** + * Returns a bitmap that is the bitwise AND NOT of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * - * @group string_funcs - * @since 3.5.0 + * @param left + * A column that evaluates to a binary bitmap. + * @param right + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary bitmap. */ - def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column): Column = - Column.fn("mask", input, upperChar, lowerChar, digitChar) + def bitmap_andnot(left: Column, right: Column): Column = + Column.fn("bitmap_andnot", left, right) /** - * Masks the given string value. This can be useful for creating copies of tables with sensitive - * information removed. + * Returns a bitmap that is the bitwise XOR of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param lowerChar - * character to replace lower-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param digitChar - * character to replace digit characters with. Specify NULL to retain original character. A - * column that evaluates to a string. - * @param otherChar - * character to replace all other characters with. Specify NULL to retain original character. - * A column that evaluates to a string. + * @param left + * A column that evaluates to a binary bitmap. + * @param right + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 + * @return + * Returns a column that evaluates to a binary bitmap. + */ + def bitmap_xor(left: Column, right: Column): Column = Column.fn("bitmap_xor", left, right) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Datasketch Functions + ////////////////////////////////////////////////////////////////////////////////////////////// + + /** + * Returns the estimated number of unique values given the binary representation of a + * Datasketches HllSketch. * - * @group string_funcs + * @param c + * The binary representation of a Datasketches HllSketch. A column that evaluates to a binary. + * @group sketch_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def mask( - input: Column, - upperChar: Column, - lowerChar: Column, - digitChar: Column, - otherChar: Column): Column = - Column.fn("mask", input, upperChar, lowerChar, digitChar, otherChar) + def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) /** - * Returns length of array or map. - * - * This function returns -1 for null input only if spark.sql.ansi.enabled is false and - * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the - * default settings, the function returns null for null input. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches HllSketch. * - * @param e - * the target column. A column that evaluates to an array or a map. - * @group collection_funcs - * @since 1.5.0 + * @param columnName + * Name of the column containing the binary representation of a Datasketches HllSketch. A + * column that evaluates to a binary. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a long. */ - def size(e: Column): Column = Column.fn("size", e) + def hll_sketch_estimate(columnName: String): Column = { + hll_sketch_estimate(Column(columnName)) + } /** - * Returns length of array or map. This is an alias of `size` function. - * - * This function returns -1 for null input only if spark.sql.ansi.enabled is false and - * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the - * default settings, the function returns null for null input. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values. * - * @param e - * the target column. A column that evaluates to an array or a map. - * @group collection_funcs + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def cardinality(e: Column): Column = Column.fn("cardinality", e) + def hll_union(c1: Column, c2: Column): Column = + Column.fn("hll_union", c1, c2) /** - * Sorts the input array for the given column in ascending order, according to the natural - * ordering of the array elements. Null elements will be placed at the beginning of the returned - * array. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values. * - * @param e - * the array column to sort. A column that evaluates to an array. - * @group array_funcs - * @since 1.5.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def sort_array(e: Column): Column = sort_array(e, asc = true) + def hll_union(columnName1: String, columnName2: String): Column = { + hll_union(Column(columnName1), Column(columnName2)) + } /** - * Sorts the input array for the given column in ascending or descending order, according to the - * natural ordering of the array elements. NaN is greater than any non-NaN elements for - * double/float type. Null elements will be placed at the beginning of the returned array in - * ascending order or at the end of the returned array in descending order. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values and + * allowDifferentLgConfigK is set to false. * - * @param e - * the array column to sort. A column that evaluates to an array. - * @param asc - * whether to sort in ascending order. A column that evaluates to a boolean. Must be a - * constant. - * @group array_funcs - * @since 1.5.0 + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def sort_array(e: Column, asc: Boolean): Column = Column.fn("sort_array", e, lit(asc)) + def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = + Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) /** - * Returns the minimum value in the array. NaN is greater than any non-NaN elements for - * double/float type. NULL elements are skipped. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values and + * allowDifferentLgConfigK is set to false. * - * @param e - * the array column. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column of the element type of the input array. + * Returns a column that evaluates to a binary. */ - def array_min(e: Column): Column = Column.fn("array_min", e) + def hll_union( + columnName1: String, + columnName2: String, + allowDifferentLgConfigK: Boolean): Column = { + hll_union(Column(columnName1), Column(columnName2), allowDifferentLgConfigK) + } /** - * Returns the maximum value in the array. NaN is greater than any non-NaN elements for - * double/float type. NULL elements are skipped. + * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches AnotB object * - * @param e - * the input column. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the element type of the input array. + * Returns a column that evaluates to a binary. */ - def array_max(e: Column): Column = Column.fn("array_max", e) + def theta_difference(c1: Column, c2: Column): Column = + Column.fn("theta_difference", c1, c2) /** - * Returns the total number of elements in the array. The function returns null for null input. + * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches AnotB object * - * @param e - * the input column. A column that evaluates to an array. - * @group array_funcs - * @since 3.5.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def array_size(e: Column): Column = Column.fn("array_size", e) + def theta_difference(columnName1: String, columnName2: String): Column = { + theta_difference(Column(columnName1), Column(columnName2)) + } /** - * Aggregate function: returns a list of objects with duplicates. + * Intersects two binary representations of Datasketches ThetaSketch objects in the input + * columns using a Datasketches Intersection object * - * @param e - * the input column. A column that evaluates to any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. - * @group agg_funcs - * @since 3.5.0 + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def array_agg(e: Column): Column = Column.fn("array_agg", e) + def theta_intersection(c1: Column, c2: Column): Column = + Column.fn("theta_intersection", c1, c2) /** - * Returns a random permutation of the given array. - * - * @param e - * the input column. A column that evaluates to an array. - * @note - * The function is non-deterministic. + * Intersects two binary representations of Datasketches ThetaSketch objects in the input + * columns using a Datasketches Intersection object * - * @group array_funcs - * @since 2.4.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def shuffle(e: Column): Column = shuffle(e, lit(SparkClassUtils.random.nextLong)) + def theta_intersection(columnName1: String, columnName2: String): Column = { + theta_intersection(Column(columnName1), Column(columnName2)) + } /** - * Returns a random permutation of the given array. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches ThetaSketch. * - * @param e - * the input column. A column that evaluates to an array. - * @param seed - * the seed for the random generator. A column that evaluates to an integral. Must be a - * constant. - * @note - * The function is non-deterministic. + * @param c + * The binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. + */ + def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) + + /** + * Returns the estimated number of unique values given the binary representation of a + * Datasketches ThetaSketch. * - * @group array_funcs - * @since 4.0.0 + * @param columnName + * Name of the column containing the binary representation of a Datasketches ThetaSketch. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a long. */ - def shuffle(e: Column, seed: Column): Column = Column.fn("shuffle", e, seed) + def theta_sketch_estimate(columnName: String): Column = { + theta_sketch_estimate(Column(columnName)) + } /** - * Returns a reversed string or an array with reverse order of elements. - * @param e - * the input column. A column that evaluates to a string, a binary, or an array. - * @group collection_funcs - * @since 1.5.0 + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It is configured with the default value of 12 for + * `lgNomEntries`. + * + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def reverse(e: Column): Column = Column.fn("reverse", e) + def theta_union(c1: Column, c2: Column): Column = + Column.fn("theta_union", c1, c2) /** - * Creates a single array from an array of arrays. If a structure of nested arrays is deeper - * than two levels, only one level of nesting is removed. - * @param e - * the input column. A column that evaluates to an array of arrays. - * @group array_funcs - * @since 2.4.0 + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It is configured with the default value of 12 for + * `lgNomEntries`. + * + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def flatten(e: Column): Column = Column.fn("flatten", e) + def theta_union(columnName1: String, columnName2: String): Column = { + theta_union(Column(columnName1), Column(columnName2)) + } /** - * Generate a sequence of integers from start to stop, incrementing by step. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param start - * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a - * date, or a timestamp. - * @param stop - * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, - * or a timestamp. - * @param step - * the value to add to the current element to get the next element. A column that evaluates to - * an integral or interval. - * @group array_funcs - * @since 2.4.0 + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def sequence(start: Column, stop: Column, step: Column): Column = - Column.fn("sequence", start, stop, step) + def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("theta_union", c1, c2, lit(lgNomEntries)) /** - * Generate a sequence of integers from start to stop, incrementing by 1 if start is less than - * or equal to stop, otherwise -1. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param start - * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a - * date, or a timestamp. - * @param stop - * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, - * or a timestamp. - * @group array_funcs - * @since 2.4.0 + * @param columnName1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param columnName2 + * The second ThetaSketch column to union. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def sequence(start: Column, stop: Column): Column = Column.fn("sequence", start, stop) + def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { + theta_union(Column(columnName1), Column(columnName2), lgNomEntries) + } /** - * Creates an array containing the left argument repeated the number of times given by the right - * argument. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param left - * the value to repeat. A column that evaluates to any type. - * @param right - * the number of times to repeat the value. A column that evaluates to an integral. - * @group array_funcs - * @since 2.4.0 + * @param c1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param c2 + * The second ThetaSketch column to union. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def array_repeat(left: Column, right: Column): Column = Column.fn("array_repeat", left, right) + def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = + Column.fn("theta_union", c1, c2, lgNomEntries) /** - * Creates an array containing the left argument repeated the number of times given by the right - * argument. + * Subtracts two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. * - * @param e - * the value to repeat. A column that evaluates to any type. - * @param count - * the number of times to repeat the value. A column that evaluates to an integral. Must be a - * constant. - * @group array_funcs - * @since 2.4.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def array_repeat(e: Column, count: Int): Column = array_repeat(e, lit(count)) + def tuple_difference_double(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_double", c1, c2) /** - * Returns true if the map contains the key. - * @param column - * the input column. A column that evaluates to a map. - * @param key - * the key to check for. A column that evaluates to the map's key type. Must be a constant. - * @group map_funcs - * @since 3.3.0 + * Subtracts two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. + * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a binary. */ - def map_contains_key(column: Column, key: Any): Column = - Column.fn("map_contains_key", column, lit(key)) + def tuple_difference_double(columnName1: String, columnName2: String): Column = + tuple_difference_double(Column(columnName1), Column(columnName2)) /** - * Returns an unordered array containing the keys of the map. - * @param e - * the input column. A column that evaluates to a map. - * @group map_funcs - * @since 2.3.0 + * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. + * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def map_keys(e: Column): Column = Column.fn("map_keys", e) + def tuple_difference_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_integer", c1, c2) /** - * Returns an unordered array containing the values of the map. - * @param e - * the input column. A column that evaluates to a map. - * @group map_funcs - * @since 2.3.0 + * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. + * + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def map_values(e: Column): Column = Column.fn("map_values", e) + def tuple_difference_integer(columnName1: String, columnName2: String): Column = + tuple_difference_integer(Column(columnName1), Column(columnName2)) /** - * Returns an unordered array of all entries in the given map. - * @param e - * the input column. A column that evaluates to a map. - * @group map_funcs - * @since 3.0.0 + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). It is configured with the default mode of 'sum'. + * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def map_entries(e: Column): Column = Column.fn("map_entries", e) + def tuple_intersection_double(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_double", c1, c2) /** - * Returns a map created from the given array of entries. - * @param e - * the array of entries to convert. A column that evaluates to an array of structs, each with - * a key and value field. - * @group map_funcs - * @since 2.4.0 + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). It is configured with the default mode of 'sum'. + * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a binary. */ - def map_from_entries(e: Column): Column = Column.fn("map_from_entries", e) + def tuple_intersection_double(columnName1: String, columnName2: String): Column = + tuple_intersection_double(Column(columnName1), Column(columnName2)) /** - * Returns a merged array of structs in which the N-th struct contains all N-th values of input - * arrays. - * @param e - * the columns of arrays to be merged. Each is a column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). + * + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - @scala.annotation.varargs - def arrays_zip(e: Column*): Column = Column.fn("arrays_zip", e: _*) + def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_double", c1, c2, lit(mode)) /** - * Returns the union of all the given maps. - * @param cols - * the maps to merge. Each is a column that evaluates to a map. - * @group map_funcs - * @since 2.4.0 + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). + * + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a binary. */ - @scala.annotation.varargs - def map_concat(cols: Column*): Column = Column.fn("map_concat", cols: _*) + def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = + tuple_intersection_double(Column(columnName1), Column(columnName2), mode) - // scalastyle:off line.size.limit /** - * Parses a column containing a CSV string into a `StructType` with the specified schema. - * Returns `null`, in the case of an unparseable string. - * - * @param e - * a string column containing CSV data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the CSV string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the CSV is parsed. accepts the same options and the CSV data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @group csv_funcs - * @since 3.0.0 + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a binary. */ - // scalastyle:on line.size.limit - def from_csv(e: Column, schema: StructType, options: Map[String, String]): Column = - from_csv(e, lit(schema.toDDL), options.iterator) + def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_double", c1, c2, mode) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a CSV string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. - * - * @param e - * a string column containing CSV data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the CSV string. A column that evaluates to a string. - * @param options - * options to control how the CSV is parsed. accepts the same options and the CSV data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). It is configured with the default mode of 'sum'. * - * @group csv_funcs - * @since 3.0.0 + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a binary. */ - // scalastyle:on line.size.limit - def from_csv(e: Column, schema: Column, options: java.util.Map[String, String]): Column = - from_csv(e, schema, options.asScala.iterator) - - private def from_csv(e: Column, schema: Column, options: Iterator[(String, String)]): Column = - Column.fnWithOptions("from_csv", options, e, schema) + def tuple_intersection_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_integer", c1, c2) /** - * Parses a CSV string and infers its schema in DDL format. - * - * @param csv - * a CSV string. A string. Must be a constant. + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). It is configured with the default mode of 'sum'. * - * @group csv_funcs - * @since 3.0.0 + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def schema_of_csv(csv: String): Column = schema_of_csv(lit(csv)) + def tuple_intersection_integer(columnName1: String, columnName2: String): Column = + tuple_intersection_integer(Column(columnName1), Column(columnName2)) /** - * Parses a CSV string and infers its schema in DDL format. - * - * @param csv - * a foldable string column containing a CSV string. A column that evaluates to a string. + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). * - * @group csv_funcs - * @since 3.0.0 + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def schema_of_csv(csv: Column): Column = schema_of_csv(csv, Collections.emptyMap()) + def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) - // scalastyle:off line.size.limit /** - * Parses a CSV string and infers its schema in DDL format using options. + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). * - * @param csv - * a foldable string column containing a CSV string. A column that evaluates to a string. - * @param options - * options to control how the CSV is parsed. accepts the same options and the CSV data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * a column with string literal containing schema in DDL format. Returns a column that - * evaluates to a string. - * @group csv_funcs - * @since 3.0.0 + * Returns a column that evaluates to a binary. */ - // scalastyle:on line.size.limit - def schema_of_csv(csv: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("schema_of_csv", options.asScala.iterator, csv) + def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = + tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) - // scalastyle:off line.size.limit /** - * (Java-specific) Converts a column containing a `StructType` into a CSV string with the - * specified schema. Throws an exception, in the case of an unsupported type. - * - * @param e - * a column containing a struct. A column that evaluates to a string. - * @param options - * options to control how the struct column is converted into a CSV string. It accepts the - * same options and the CSV data source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). * - * @group csv_funcs - * @since 3.0.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - // scalastyle:on line.size.limit - def to_csv(e: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("to_csv", options.asScala.iterator, e) + def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_integer", c1, c2, mode) /** - * Converts a column containing a `StructType` into a CSV string with the specified schema. - * Throws an exception, in the case of an unsupported type. - * - * @param e - * a column containing a struct. A column that evaluates to a string. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with double summary data type. * - * @group csv_funcs - * @since 3.0.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) + def tuple_sketch_estimate_double(c: Column): Column = + Column.fn("tuple_sketch_estimate_double", c) - // scalastyle:off line.size.limit /** - * Parses a column containing a XML string into the data type corresponding to the specified - * schema. Returns `null`, in the case of an unparseable string. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with double summary data type. * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the XML is parsed. accepts the same options and the XML data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = - from_xml(e, lit(schema.sql), options.asScala.iterator) + def tuple_sketch_estimate_double(columnName: String): Column = + tuple_sketch_estimate_double(Column(columnName)) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a XML string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with integer summary data type. * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. - * @param options - * options to control how the XML is parsed. accepts the same options and the xml data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { - from_xml(e, lit(schema), options) - } + def tuple_sketch_estimate_integer(c: Column): Column = + Column.fn("tuple_sketch_estimate_integer", c) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a XML string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with integer summary data type. * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A column that evaluates to a string. - * @group xml_funcs - * @since 4.0.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: Column): Column = { - from_xml(e, schema, Iterator.empty) - } + def tuple_sketch_estimate_integer(columnName: String): Column = + tuple_sketch_estimate_integer(Column(columnName)) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a XML string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A column that evaluates to a string. - * @param options - * options to control how the XML is parsed. accepts the same options and the XML data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = - from_xml(e, schema, options.asScala.iterator) + def tuple_sketch_summary_double(c: Column): Column = + Column.fn("tuple_sketch_summary_double", c) /** - * Parses a column containing a XML string into the data type corresponding to the specified - * schema. Returns `null`, in the case of an unparseable string. - * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a - * constant. + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @group xml_funcs - * @since 4.0.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - def from_xml(e: Column, schema: StructType): Column = - from_xml(e, schema, Map.empty[String, String].asJava) - - private def from_xml(e: Column, schema: Column, options: Iterator[(String, String)]): Column = { - Column.fnWithOptions("from_xml", options, e, schema) - } + def tuple_sketch_summary_double(columnName: String): Column = + tuple_sketch_summary_double(Column(columnName)) /** - * Parses a XML string and infers its schema in DDL format. + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param xml - * a XML string. A string. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) + def tuple_sketch_summary_double(c: Column, mode: String): Column = + Column.fn("tuple_sketch_summary_double", c, lit(mode)) /** - * Parses a XML string and infers its schema in DDL format. + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param xml - * a foldable string column containing a XML string. A column that evaluates to a string. - * @group xml_funcs - * @since 4.0.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) - - // scalastyle:off line.size.limit + def tuple_sketch_summary_double(columnName: String, mode: String): Column = + tuple_sketch_summary_double(Column(columnName), mode) /** - * Parses a XML string and infers its schema in DDL format using options. + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param xml - * a foldable string column containing XML data. A column that evaluates to a string. - * @param options - * options to control how the xml is parsed. accepts the same options and the XML data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * a column with string literal containing schema in DDL format. Returns a column that - * evaluates to a string. - * @group xml_funcs - * @since 4.0.0 + * Returns a column that evaluates to a double. */ - // scalastyle:on line.size.limit - def schema_of_xml(xml: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("schema_of_xml", options.asScala.iterator, xml) - - // scalastyle:off line.size.limit + def tuple_sketch_summary_double(c: Column, mode: Column): Column = + Column.fn("tuple_sketch_summary_double", c, mode) /** - * (Java-specific) Converts a column containing a `StructType` into a XML string with the - * specified schema. Throws an exception, in the case of an unsupported type. + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @param e - * a column containing a struct. A column that evaluates to a string. - * @param options - * options to control how the struct column is converted into a XML string. It accepts the - * same options as the XML data source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - // scalastyle:on line.size.limit - def to_xml(e: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("to_xml", options.asScala.iterator, e) + def tuple_sketch_summary_integer(c: Column): Column = + Column.fn("tuple_sketch_summary_integer", c) /** - * Converts a column containing a `StructType` into a XML string with the specified schema. - * Throws an exception, in the case of an unsupported type. + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @param e - * a column containing a struct. A column that evaluates to a string. - * @group xml_funcs - * @since 4.0.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) + def tuple_sketch_summary_integer(columnName: String): Column = + tuple_sketch_summary_integer(Column(columnName)) /** - * (Java-specific) A transform for timestamps and dates to partition data into years. + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param e - * the target column to transform. A column that evaluates to a date or a timestamp. - * @group partition_transforms - * @since 3.0.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ - def years(e: Column): Column = partitioning.years(e) + def tuple_sketch_summary_integer(c: Column, mode: String): Column = + Column.fn("tuple_sketch_summary_integer", c, lit(mode)) /** - * (Java-specific) A transform for timestamps and dates to partition data into months. + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param e - * the target column to transform. A column that evaluates to a date or a timestamp. - * @group partition_transforms - * @since 3.0.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ - def months(e: Column): Column = partitioning.months(e) + def tuple_sketch_summary_integer(columnName: String, mode: String): Column = + tuple_sketch_summary_integer(Column(columnName), mode) /** - * (Java-specific) A transform for timestamps and dates to partition data into days. + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param e - * the target column to transform. A column that evaluates to a date or a timestamp. - * @group partition_transforms - * @since 3.0.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a long. */ - def days(e: Column): Column = partitioning.days(e) + def tuple_sketch_summary_integer(c: Column, mode: Column): Column = + Column.fn("tuple_sketch_summary_integer", c, mode) /** - * Returns a string array of values within the nodes of xml that match the XPath expression. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a double. */ - def xpath(xml: Column, path: Column): Column = - Column.fn("xpath", xml, path) + def tuple_sketch_theta_double(c: Column): Column = + Column.fn("tuple_sketch_theta_double", c) /** - * Returns true if the XPath expression evaluates to true, or if a matching node is found. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a double. */ - def xpath_boolean(xml: Column, path: Column): Column = - Column.fn("xpath_boolean", xml, path) + def tuple_sketch_theta_double(columnName: String): Column = + tuple_sketch_theta_double(Column(columnName)) /** - * Returns a double value, the value zero if no match is found, or NaN if a match is found but - * the value is non-numeric. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a double. */ - def xpath_double(xml: Column, path: Column): Column = - Column.fn("xpath_double", xml, path) + def tuple_sketch_theta_integer(c: Column): Column = + Column.fn("tuple_sketch_theta_integer", c) /** - * Returns a double value, the value zero if no match is found, or NaN if a match is found but - * the value is non-numeric. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a double. */ - def xpath_number(xml: Column, path: Column): Column = - Column.fn("xpath_number", xml, path) + def tuple_sketch_theta_integer(columnName: String): Column = + tuple_sketch_theta_integer(Column(columnName)) /** - * Returns a float value, the value zero if no match is found, or NaN if a match is found but - * the value is non-numeric. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a float. + * Returns a column that evaluates to a binary. */ - def xpath_float(xml: Column, path: Column): Column = - Column.fn("xpath_float", xml, path) + def tuple_union_double(c1: Column, c2: Column): Column = + Column.fn("tuple_union_double", c1, c2) /** - * Returns an integer value, or the value zero if no match is found, or a match is found but the - * value is non-numeric. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def xpath_int(xml: Column, path: Column): Column = - Column.fn("xpath_int", xml, path) + def tuple_union_double(columnName1: String, columnName2: String): Column = + tuple_union_double(Column(columnName1), Column(columnName2)) /** - * Returns a long integer value, or the value zero if no match is found, or a match is found but - * the value is non-numeric. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def xpath_long(xml: Column, path: Column): Column = - Column.fn("xpath_long", xml, path) + def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) /** - * Returns a short integer value, or the value zero if no match is found, or a match is found - * but the value is non-numeric. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a short. + * Returns a column that evaluates to a binary. */ - def xpath_short(xml: Column, path: Column): Column = - Column.fn("xpath_short", xml, path) + def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = + tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) /** - * Returns the text contents of the first xml node that matches the XPath expression. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def xpath_string(xml: Column, path: Column): Column = - Column.fn("xpath_string", xml, path) + def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) /** - * (Java-specific) A transform for timestamps to partition data into hours. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param e - * target date or timestamp column to work on. A column that evaluates to a date or timestamp. - * @group partition_transforms - * @since 3.0.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ - def hours(e: Column): Column = partitioning.hours(e) + def tuple_union_double( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to - * `targetTz`. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param sourceTz - * the time zone for the input timestamp. If it is missed, the current session time zone is - * used as the source time zone. A column that evaluates to a string. - * @param targetTz - * the time zone to which the input timestamp should be converted. A column that evaluates to - * a string. - * @param sourceTs - * a timestamp without time zone. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = - Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) + def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) /** - * Converts the timestamp without time zone `sourceTs` from the current time zone to `targetTz`. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param targetTz - * the time zone to which the input timestamp should be converted. A column that evaluates to - * a string. - * @param sourceTs - * a timestamp without time zone. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def convert_timezone(targetTz: Column, sourceTs: Column): Column = - Column.fn("convert_timezone", targetTz, sourceTs) + def tuple_union_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_union_integer", c1, c2) /** - * Make DayTimeIntervalType duration from days, hours, mins and secs. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @param secs - * the number of seconds with the fractional part in microsecond precision. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = - Column.fn("make_dt_interval", days, hours, mins, secs) + def tuple_union_integer(columnName1: String, columnName2: String): Column = + tuple_union_integer(Column(columnName1), Column(columnName2)) /** - * Make DayTimeIntervalType duration from days, hours and mins. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_dt_interval(days: Column, hours: Column, mins: Column): Column = - Column.fn("make_dt_interval", days, hours, mins) + def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) /** - * Make DayTimeIntervalType duration from days and hours. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_dt_interval(days: Column, hours: Column): Column = - Column.fn("make_dt_interval", days, hours) + def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = + tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) /** - * Make DayTimeIntervalType duration from days. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_dt_interval(days: Column): Column = - Column.fn("make_dt_interval", days) + def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) /** - * Make DayTimeIntervalType duration. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_dt_interval(): Column = - Column.fn("make_dt_interval") + def tuple_union_integer( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @param secs - * the number of seconds with the fractional part in microsecond precision. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def try_make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days, hours, mins, secs) + def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) /** - * Make interval from years, months, weeks, days, hours, mins and secs. + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * double summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @param secs - * the number of seconds with the fractional part in microsecond precision. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_interval", years, months, weeks, days, hours, mins, secs) + def tuple_difference_theta_double(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_theta_double", c1, c2) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * double summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def try_make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days, hours, mins) + def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = + tuple_difference_theta_double(Column(columnName1), Column(columnName2)) /** - * Make interval from years, months, weeks, days, hours and mins. + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * integer summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column): Column = - Column.fn("make_interval", years, months, weeks, days, hours, mins) + def tuple_difference_theta_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_theta_integer", c1, c2) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * integer summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def try_make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days, hours) + def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = + tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) /** - * Make interval from years, months, weeks, days and hours. - * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column): Column = - Column.fn("make_interval", years, months, weeks, days, hours) + def tuple_intersection_theta_double(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_theta_double", c1, c2) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days) + def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = + tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) /** - * Make interval from years, months, weeks and days. + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = - Column.fn("make_interval", years, months, weeks, days) + def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def try_make_interval(years: Column, months: Column, weeks: Column): Column = - Column.fn("try_make_interval", years, months, weeks) + def tuple_intersection_theta_double( + columnName1: String, + columnName2: String, + mode: String): Column = + tuple_intersection_theta_double(Column(columnName1), Column(columnName2), mode) /** - * Make interval from years, months and weeks. + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * The number of weeks, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval(years: Column, months: Column, weeks: Column): Column = - Column.fn("make_interval", years, months, weeks) + def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_theta_double", c1, c2, mode) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def try_make_interval(years: Column, months: Column): Column = - Column.fn("try_make_interval", years, months) + def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_theta_integer", c1, c2) /** - * Make interval from years and months. + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval(years: Column, months: Column): Column = - Column.fn("make_interval", years, months) + def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = + tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def try_make_interval(years: Column): Column = - Column.fn("try_make_interval", years) + def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) /** - * Make interval from years. + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval(years: Column): Column = - Column.fn("make_interval", years) + def tuple_intersection_theta_integer( + columnName1: String, + columnName2: String, + mode: String): Column = + tuple_intersection_theta_integer(Column(columnName1), Column(columnName2), mode) /** - * Make interval. + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_interval(): Column = - Column.fn("make_interval") + def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_theta_integer", c1, c2, mode) /** - * Create timestamp from years, months, days, hours, mins, secs and timezone fields. The result - * data type is consistent with the value of configuration `spark.sql.timestampType`. If the - * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. - * Otherwise, it will throw an error instead. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("make_timestamp", years, months, days, hours, mins, secs, timezone) + def tuple_union_theta_double(c1: Column, c2: Column): Column = + Column.fn("tuple_union_theta_double", c1, c2) /** - * Create timestamp from years, months, days, hours, mins and secs fields. The result data type - * is consistent with the value of configuration `spark.sql.timestampType`. If the configuration - * `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it - * will throw an error instead. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_timestamp", years, months, days, hours, mins, secs) + def tuple_union_theta_double(columnName1: String, columnName2: String): Column = + tuple_union_theta_double(Column(columnName1), Column(columnName2)) /** - * Create a local date-time from date, time, and timezone fields. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp(date: Column, time: Column, timezone: Column): Column = - Column.fn("make_timestamp", date, time, timezone) + def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) /** - * Create a local date-time from date and time fields. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp(date: Column, time: Column): Column = - Column.fn("make_timestamp", date, time) + def tuple_union_theta_double( + columnName1: String, + columnName2: String, + lgNomEntries: Int): Column = + tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries) /** - * Try to create a timestamp from years, months, days, hours, mins, secs and timezone fields. - * The result data type is consistent with the value of configuration `spark.sql.timestampType`. - * The function returns NULL on invalid inputs. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.0.0 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("try_make_timestamp", years, months, days, hours, mins, secs, timezone) + def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) /** - * Try to create a timestamp from years, months, days, hours, mins, and secs fields. The result - * data type is consistent with the value of configuration `spark.sql.timestampType`. The - * function returns NULL on invalid inputs. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_timestamp", years, months, days, hours, mins, secs) + def tuple_union_theta_double( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * Try to create a local date-time from date, time, and timezone fields. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = - Column.fn("try_make_timestamp", date, time, timezone) + def tuple_union_theta_double( + c1: Column, + c2: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_union_theta_double", c1, c2, lgNomEntries, mode) /** - * Try to create a local date-time from date and time fields. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp(date: Column, time: Column): Column = - Column.fn("try_make_timestamp", date, time) + def tuple_union_theta_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_union_theta_integer", c1, c2) /** - * Create the current timestamp with local time zone from years, months, days, hours, mins, secs - * and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function - * returns NULL on invalid inputs. Otherwise, it will throw an error instead. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) + def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = + tuple_union_theta_integer(Column(columnName1), Column(columnName2)) /** - * Create the current timestamp with local time zone from years, months, days, hours, mins and - * secs fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns - * NULL on invalid inputs. Otherwise, it will throw an error instead. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs) + def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) /** - * Try to create the current timestamp with local time zone from years, months, days, hours, - * mins, secs and timezone fields. The function returns NULL on invalid inputs. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.0.0 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) + def tuple_union_theta_integer( + columnName1: String, + columnName2: String, + lgNomEntries: Int): Column = + tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries) /** - * Try to create the current timestamp with local time zone from years, months, days, hours, - * mins and secs fields. The function returns NULL on invalid inputs. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs) + def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) /** - * Create local date-time from years, months, days, hours, mins, secs fields. If the - * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. - * Otherwise, it will throw an error instead. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp_ntz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_timestamp_ntz", years, months, days, hours, mins, secs) + def tuple_union_theta_integer( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * Create a local date-time from date and time fields. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def make_timestamp_ntz(date: Column, time: Column): Column = - Column.fn("make_timestamp_ntz", date, time) + def tuple_union_theta_integer( + c1: Column, + c2: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_union_theta_integer", c1, c2, lgNomEntries, mode) /** - * Try to create a local date-time from years, months, days, hours, mins, secs fields. The - * function returns NULL on invalid inputs. + * Returns a string with human readable summary information about the KLL bigint sketch. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def try_make_timestamp_ntz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_timestamp_ntz", years, months, days, hours, mins, secs) + def kll_sketch_to_string_bigint(e: Column): Column = + Column.fn("kll_sketch_to_string_bigint", e) /** - * Try to create a local date-time from date and time fields. + * Returns a string with human readable summary information about the KLL float sketch. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs + * @param e + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs * @since 4.1.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def try_make_timestamp_ntz(date: Column, time: Column): Column = - Column.fn("try_make_timestamp_ntz", date, time) + def kll_sketch_to_string_float(e: Column): Column = + Column.fn("kll_sketch_to_string_float", e) /** - * Make year-month interval from years, months. + * Returns a string with human readable summary information about the KLL double sketch. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def make_ym_interval(years: Column, months: Column): Column = - Column.fn("make_ym_interval", years, months) + def kll_sketch_to_string_double(e: Column): Column = + Column.fn("kll_sketch_to_string_double", e) /** - * Make year-month interval from years. + * Returns the number of items collected in the KLL bigint sketch. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a long. */ - def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) + def kll_sketch_get_n_bigint(e: Column): Column = + Column.fn("kll_sketch_get_n_bigint", e) /** - * Make year-month interval. + * Returns the number of items collected in the KLL float sketch. * - * @group datetime_funcs - * @since 3.5.0 + * @param e + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a long. */ - def make_ym_interval(): Column = Column.fn("make_ym_interval") + def kll_sketch_get_n_float(e: Column): Column = + Column.fn("kll_sketch_get_n_float", e) /** - * (Java-specific) A transform for any type that partitions by a hash of the input column. + * Returns the number of items collected in the KLL double sketch. * - * @param numBuckets - * The number of buckets. A column that evaluates to an integral. Must be a constant. * @param e - * The input column to partition. A column of any type. - * @group partition_transforms - * @since 3.0.0 + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a long. */ - def bucket(numBuckets: Column, e: Column): Column = partitioning.bucket(numBuckets, e) + def kll_sketch_get_n_double(e: Column): Column = + Column.fn("kll_sketch_get_n_double", e) /** - * (Java-specific) A transform for any type that partitions by a hash of the input column. + * Merges two KLL bigint sketch buffers together into one. * - * @param numBuckets - * The number of buckets. Must be a constant. - * @param e - * The input column to partition. A column of any type. - * @group partition_transforms - * @since 3.0.0 + * @param left + * The first KLL bigint sketch. A column that evaluates to a binary. + * @param right + * The second KLL bigint sketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ - def bucket(numBuckets: Int, e: Column): Column = partitioning.bucket(numBuckets, e) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Predicates functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def kll_sketch_merge_bigint(left: Column, right: Column): Column = + Column.fn("kll_sketch_merge_bigint", left, right) /** - * Returns `col2` if `col1` is null, or `col1` otherwise. + * Merges two KLL float sketch buffers together into one. * - * @param col1 - * The column to test for null. A column of any type. - * @param col2 - * The column to return when col1 is null. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @param left + * The first KLL float sketch. A column that evaluates to a binary. + * @param right + * The second KLL float sketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) + def kll_sketch_merge_float(left: Column, right: Column): Column = + Column.fn("kll_sketch_merge_float", left, right) /** - * Returns true if `col` is not null, or false otherwise. + * Merges two KLL double sketch buffers together into one. * - * @param col - * The column to check. A column of any type. - * @group predicate_funcs - * @since 3.5.0 + * @param left + * The first KLL double sketch. A column that evaluates to a binary. + * @param right + * The second KLL double sketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a binary. */ - def isnotnull(col: Column): Column = Column.fn("isnotnull", col) + def kll_sketch_merge_double(left: Column, right: Column): Column = + Column.fn("kll_sketch_merge_double", left, right) /** - * Returns same result as the EQUAL(=) operator for non-null operands, but returns true if both - * are null, false if one of the them is null. + * Extracts a quantile value from a KLL bigint sketch given an input rank value. The rank can be + * a single value or an array. * - * @param col1 - * The first column to compare. A column of any type. - * @param col2 - * The second column to compare. A column of any type. - * @group predicate_funcs - * @since 3.5.0 + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. Must be a constant. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a long, or an array of longs when `rank` is an array. */ - def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) + def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = + Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) /** - * Returns null if `col1` equals to `col2`, or `col1` otherwise. + * Extracts a quantile value from a KLL float sketch given an input rank value. The rank can be + * a single value or an array. * - * @param col1 - * The value to return if it is not equal to `col2`. A column of any type. - * @param col2 - * The value compared with `col1`. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a float, or an array of floats when `rank` is an array. */ - def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) + def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = + Column.fn("kll_sketch_get_quantile_float", sketch, rank) /** - * Returns null if `col` is equal to zero, or `col` otherwise. + * Extracts a quantile value from a KLL double sketch given an input rank value. The rank can be + * a single value or an array. * - * @param col - * The input value. A column that evaluates to a numeric. - * @group conditional_funcs - * @since 4.0.0 + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double, or an array of doubles when `rank` is an + * array. */ - def nullifzero(col: Column): Column = Column.fn("nullifzero", col) + def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = + Column.fn("kll_sketch_get_quantile_double", sketch, rank) /** - * Returns `col2` if `col1` is null, or `col1` otherwise. + * Extracts a rank value from a KLL bigint sketch given an input quantile value. The quantile + * can be a single value or an array. * - * @param col1 - * The value to return if it is not null. A column of any type. - * @param col2 - * The value to return if `col1` is null. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to an integral or an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ - def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) + def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = + Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) /** - * Returns `col2` if `col1` is not null, or `col3` otherwise. + * Extracts a rank value from a KLL float sketch given an input quantile value. The quantile can + * be a single value or an array. * - * @param col1 - * The value that determines which branch to return. A column of any type. - * @param col2 - * The value to return if `col1` is not null. A column of any type. - * @param col3 - * The value to return if `col1` is null. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to a numeric or an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ - def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) + def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = + Column.fn("kll_sketch_get_rank_float", sketch, quantile) /** - * Returns zero if `col` is null, or `col` otherwise. + * Extracts a rank value from a KLL double sketch given an input quantile value. The quantile + * can be a single value or an array. * - * @param col - * The input value. A column that evaluates to a numeric. - * @group conditional_funcs - * @since 4.0.0 + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to look up. A column that evaluates to a numeric or an array. Must be + * a constant. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. - */ - def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) - - // scalastyle:off line.size.limit - // scalastyle:off parameter.number - - /* Use the following code to generate: - - (0 to 10).foreach { x => - val types = (1 to x).foldRight("RT")((i, s) => s"A$i, $s") - val typeSeq = "RT" +: (1 to x).map(i => s"A$i") - val typeTags = typeSeq.map(t => s"$t: TypeTag").mkString(", ") - val implicitTypeTags = typeSeq.map(t => s"implicitly[TypeTag[$t]]").mkString(", ") - println(s""" - |/** - | * Defines a Scala closure of $x arguments as user-defined function (UDF). - | * The data types are automatically inferred based on the Scala closure's - | * signature. By default the returned UDF is deterministic. To change it to - | * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`. - | * - | * @group udf_funcs - | * @since 1.3.0 - | */ - |def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = { - | SparkUserDefinedFunction(f, $implicitTypeTags) - |}""".stripMargin) - } - - (0 to 10).foreach { i => - val extTypeArgs = (0 to i).map(_ => "_").mkString(", ") - println(s""" - |/** - | * Defines a Java UDF$i instance as user-defined function (UDF). - | * The caller must specify the output data type, and there is no automatic input type coercion. - | * By default the returned UDF is deterministic. To change it to nondeterministic, call the - | * API `UserDefinedFunction.asNondeterministic()`. - | * - | * @group udf_funcs - | * @since 2.3.0 - | */ - |def udf(f: UDF$i[$extTypeArgs], returnType: DataType): UserDefinedFunction = { - | SparkUserDefinedFunction(ToScalaUDF(f), returnType, $i) - |}""".stripMargin) - } - + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ + def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = + Column.fn("kll_sketch_get_rank_double", sketch, quantile) ////////////////////////////////////////////////////////////////////////////////////////////// - // ST geospatial functions + // Geospatial ST Functions ////////////////////////////////////////////////////////////////////////////////////////////// /** @@ -17261,27 +17336,191 @@ object functions { * @group st_funcs * @since 4.1.0 */ - def st_setsrid(geo: Column, srid: Int): Column = - Column.fn("st_setsrid", geo, lit(srid)) + def st_setsrid(geo: Column, srid: Int): Column = + Column.fn("st_setsrid", geo, lit(srid)) + + /** + * Returns the SRID of the input GEOGRAPHY or GEOMETRY value. + * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @group st_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to an integer. + */ + def st_srid(geo: Column): Column = + Column.fn("st_srid", geo) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Vector Functions + ////////////////////////////////////////////////////////////////////////////////////////////// + + /** + * Returns the cosine similarity between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_cosine_similarity(left: Column, right: Column): Column = + Column.fn("vector_cosine_similarity", left, right) + + /** + * Returns the inner product (dot product) between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_inner_product(left: Column, right: Column): Column = + Column.fn("vector_inner_product", left, right) + + /** + * Returns the Euclidean (L2) distance between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_l2_distance(left: Column, right: Column): Column = + Column.fn("vector_l2_distance", left, right) + + /** + * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_norm(vector: Column, degree: Column): Column = + Column.fn("vector_norm", vector, degree) + + /** + * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_norm(vector: Column): Column = + Column.fn("vector_norm", vector) + + /** + * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def vector_normalize(vector: Column, degree: Column): Column = + Column.fn("vector_normalize", vector, degree) + + /** + * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def vector_normalize(vector: Column): Column = + Column.fn("vector_normalize", vector) + + /** + * Aggregate function: returns the element-wise mean of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def vector_avg(col: Column): Column = Column.fn("vector_avg", col) /** - * Returns the SRID of the input GEOGRAPHY or GEOMETRY value. - * - * @param geo - * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a - * geography or geometry. - * @group st_funcs - * @since 4.1.0 + * Aggregate function: returns the element-wise sum of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def st_srid(geo: Column): Column = - Column.fn("st_srid", geo) + def vector_sum(col: Column): Column = Column.fn("vector_sum", col) ////////////////////////////////////////////////////////////////////////////////////////////// - // Scala UDF functions + // UDF, UDAF and UDT ////////////////////////////////////////////////////////////////////////////////////////////// + // scalastyle:off line.size.limit + // scalastyle:off parameter.number + + /* Use the following code to generate: + + (0 to 10).foreach { x => + val types = (1 to x).foldRight("RT")((i, s) => s"A$i, $s") + val typeSeq = "RT" +: (1 to x).map(i => s"A$i") + val typeTags = typeSeq.map(t => s"$t: TypeTag").mkString(", ") + val implicitTypeTags = typeSeq.map(t => s"implicitly[TypeTag[$t]]").mkString(", ") + println(s""" + |/** + | * Defines a Scala closure of $x arguments as user-defined function (UDF). + | * The data types are automatically inferred based on the Scala closure's + | * signature. By default the returned UDF is deterministic. To change it to + | * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`. + | * + | * @group udf_funcs + | * @since 1.3.0 + | */ + |def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = { + | SparkUserDefinedFunction(f, $implicitTypeTags) + |}""".stripMargin) + } + + (0 to 10).foreach { i => + val extTypeArgs = (0 to i).map(_ => "_").mkString(", ") + println(s""" + |/** + | * Defines a Java UDF$i instance as user-defined function (UDF). + | * The caller must specify the output data type, and there is no automatic input type coercion. + | * By default the returned UDF is deterministic. To change it to nondeterministic, call the + | * API `UserDefinedFunction.asNondeterministic()`. + | * + | * @group udf_funcs + | * @since 2.3.0 + | */ + |def udf(f: UDF$i[$extTypeArgs], returnType: DataType): UserDefinedFunction = { + | SparkUserDefinedFunction(ToScalaUDF(f), returnType, $i) + |}""".stripMargin) + } + + */ + /** * Obtains a `UserDefinedFunction` that wraps the given `Aggregator` so that it may be used with * untyped Data Frames. @@ -17624,10 +17863,6 @@ object functions { implicitly[TypeTag[A10]]) } - ////////////////////////////////////////////////////////////////////////////////////////////// - // Java UDF functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Defines a Java UDF0 instance as user-defined function (UDF). The caller must specify the * output data type, and there is no automatic input type coercion. By default the returned UDF @@ -17835,21 +18070,6 @@ object functions { @scala.annotation.varargs def call_udf(udfName: String, cols: Column*): Column = call_function(udfName, cols: _*) - /** - * Call a SQL function. - * - * @param funcName - * function name that follows the SQL identifier syntax (can be quoted, can be qualified) - * @param cols - * the expression parameters of function - * @group normal_funcs - * @since 3.5.0 - */ - @scala.annotation.varargs - def call_function(funcName: String, cols: Column*): Column = { - Column(internal.UnresolvedFunction(funcName, cols.map(_.node), isUserDefinedFunction = true)) - } - /** * Unwrap UDT data type column into its underlying type. * @param column @@ -17884,177 +18104,4 @@ object functions { def wrap_udt(column: Column, udt: Column): Column = { Column.internalFn("wrap_udt", column, udt) } - - // ---------------------- Vector Functions ---------------------- - - /** - * Returns the cosine similarity between two float vectors. - * @param left - * first vector column. A column that evaluates to an array. - * @param right - * second vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_cosine_similarity(left: Column, right: Column): Column = - Column.fn("vector_cosine_similarity", left, right) - - /** - * Returns the inner product (dot product) between two float vectors. - * @param left - * first vector column. A column that evaluates to an array. - * @param right - * second vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_inner_product(left: Column, right: Column): Column = - Column.fn("vector_inner_product", left, right) - - /** - * Returns the Euclidean (L2) distance between two float vectors. - * @param left - * first vector column. A column that evaluates to an array. - * @param right - * second vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_l2_distance(left: Column, right: Column): Column = - Column.fn("vector_l2_distance", left, right) - - /** - * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. - * @param vector - * input vector column. A column that evaluates to an array. - * @param degree - * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_norm(vector: Column, degree: Column): Column = - Column.fn("vector_norm", vector, degree) - - /** - * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). - * @param vector - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_norm(vector: Column): Column = - Column.fn("vector_norm", vector) - - /** - * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. - * @param vector - * input vector column. A column that evaluates to an array. - * @param degree - * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to an array. - */ - def vector_normalize(vector: Column, degree: Column): Column = - Column.fn("vector_normalize", vector, degree) - - /** - * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). - * @param vector - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to an array. - */ - def vector_normalize(vector: Column): Column = - Column.fn("vector_normalize", vector) - - /** - * Aggregate function: returns the element-wise mean of float vectors in a group. - * @param col - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to an array. - */ - def vector_avg(col: Column): Column = Column.fn("vector_avg", col) - - /** - * Aggregate function: returns the element-wise sum of float vectors in a group. - * @param col - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to an array. - */ - def vector_sum(col: Column): Column = Column.fn("vector_sum", col) - - // scalastyle:off - // TODO(SPARK-45970): Use @static annotation so Java can access to those - // API in the same way. Once we land this fix, should deprecate - // functions.hours, days, months, years and bucket. - object partitioning { - // scalastyle:on - /** - * (Scala-specific) A transform for timestamps and dates to partition data into years. - * - * @group partition_transforms - * @since 4.0.0 - */ - def years(e: Column): Column = Column.internalFn("years", e) - - /** - * (Scala-specific) A transform for timestamps and dates to partition data into months. - * - * @group partition_transforms - * @since 4.0.0 - */ - def months(e: Column): Column = Column.internalFn("months", e) - - /** - * (Scala-specific) A transform for timestamps and dates to partition data into days. - * - * @group partition_transforms - * @since 4.0.0 - */ - def days(e: Column): Column = Column.internalFn("days", e) - - /** - * (Scala-specific) A transform for timestamps to partition data into hours. - * - * @group partition_transforms - * @since 4.0.0 - */ - def hours(e: Column): Column = Column.internalFn("hours", e) - - /** - * (Scala-specific) A transform for any type that partitions by a hash of the input column. - * - * @group partition_transforms - * @since 4.0.0 - */ - def bucket(numBuckets: Column, e: Column): Column = Column.internalFn("bucket", numBuckets, e) - - /** - * (Scala-specific) A transform for any type that partitions by a hash of the input column. - * - * @group partition_transforms - * @since 4.0.0 - */ - def bucket(numBuckets: Int, e: Column): Column = bucket(lit(numBuckets), e) - } } From 6eba26c45cf3defe1ae8ceafa0f9b068a44e46d2 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 2 Sep 2026 13:22:08 +0000 Subject: [PATCH 3/4] Revert "[SPARK-59170][SQL] Align Scala and Python function sections" This reverts commit 5df40d21a2c74d4cf9032a0920c5b7e6abe46dca. --- python/pyspark/sql/functions/builtin.py | 42017 ++++++++-------- .../org/apache/spark/sql/functions.scala | 24253 +++++---- 2 files changed, 33092 insertions(+), 33178 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index ae1632d18da2d..54a55f91cd9fb 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -109,6 +109,11 @@ # even though there might be few exceptions for legacy or inevitable reasons. # If you are fixing other language APIs together, also please note that Scala side is not the case # since it requires making every single overridden definition. +# Public function groups are defined by pyspark.sql.functions.__all__ and mirrored in the API +# reference. +# Section headings in this implementation file are only navigation aids. + + def _get_jvm_function(name: str, sc: "SparkContext") -> Callable: """ Retrieves JVM function identified by name from @@ -171,8 +176,6 @@ def _options_to_str(options: Optional[Mapping[str, Any]] = None) -> Mapping[str, return {key: _to_str(value) for (key, value) in options.items()} return {} -# ---------------------- Normal Functions ---------------------- - @_try_remote_functions def lit(col: Any) -> Column: @@ -354,535 +357,693 @@ def col(col: str) -> Column: @_try_remote_functions -def broadcast(df: "DataFrame") -> "DataFrame": +def asc(col: "ColumnOrName") -> Column: """ - Marks a DataFrame as small enough for use in broadcast joins. + Returns a sort expression for the target column in ascending order. + This function is used in `sort` and `orderBy` functions. - .. versionadded:: 1.6.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + Target column to sort by in the ascending order. + Returns ------- - :class:`~pyspark.sql.DataFrame` - DataFrame marked as ready for broadcast join. + :class:`~pyspark.sql.Column` + The column specifying the sort order. + + See Also + -------- + :meth:`pyspark.sql.functions.asc_nulls_first` + :meth:`pyspark.sql.functions.asc_nulls_last` Examples -------- + Example 1: Sort DataFrame by 'id' column in ascending order. + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") - >>> df_small = spark.range(3) - >>> df_b = sf.broadcast(df_small) - >>> df.join(df_b, df.value == df_small.id).show() - +-----+---+ - |value| id| - +-----+---+ - | 1| 1| - | 2| 2| - +-----+---+ - """ - from py4j.java_gateway import JVMView + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(sf.asc("id")).show() + +---+-----+ + | id|value| + +---+-----+ + | 2| C| + | 3| A| + | 4| B| + +---+-----+ - from pyspark.sql.dataframe import DataFrame + Example 2: Use `asc` in `orderBy` function to sort the DataFrame. - sc = _get_active_spark_context() - return DataFrame(cast(JVMView, sc._jvm).functions.broadcast(df._jdf), df.sparkSession) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.orderBy(sf.asc("value")).show() + +---+-----+ + | id|value| + +---+-----+ + | 3| A| + | 4| B| + | 2| C| + +---+-----+ + + Example 3: Combine `asc` with `desc` to sort by multiple columns. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], + ... ['id', 'group', 'value']) + >>> df.sort(sf.asc("group"), sf.desc("value")).show() + +---+-----+-----+ + | id|group|value| + +---+-----+-----+ + | 2| A| 4| + | 3| A| 2| + | 1| B| 3| + +---+-----+-----+ + + Example 4: Implement `asc` from column expression. + + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(df.id.asc()).show() + +---+-----+ + | id|value| + +---+-----+ + | 2| C| + | 3| A| + | 4| B| + +---+-----+ + """ + return col.asc() if isinstance(col, Column) else _invoke_function("asc", col) @_try_remote_functions -def expr(str: str) -> Column: - """Parses the expression string into the column that it represents +def desc(col: "ColumnOrName") -> Column: + """ + Returns a sort expression for the target column in descending order. + This function is used in `sort` and `orderBy` functions. - .. versionadded:: 1.5.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - str : expression string - expression defined in string. + col : :class:`~pyspark.sql.Column` or column name + Target column to sort by in the descending order. Returns ------- :class:`~pyspark.sql.Column` - column representing the expression. + The column specifying the sort order. - Examples + See Also -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([["Alice"], ["Bob"]], ["name"]) - >>> df.select("*", sf.expr("length(name)")).show() - +-----+------------+ - | name|length(name)| - +-----+------------+ - |Alice| 5| - | Bob| 3| - +-----+------------+ - """ - return _invoke_function("expr", str) - + :meth:`pyspark.sql.functions.desc_nulls_first` + :meth:`pyspark.sql.functions.desc_nulls_last` -@_try_remote_functions -def call_function(funcName: str, *cols: "ColumnOrName") -> Column: - """ - Call a SQL function. + Examples + -------- + Example 1: Sort DataFrame by 'id' column in descending order. - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(sf.desc("id")).show() + +---+-----+ + | id|value| + +---+-----+ + | 4| B| + | 3| A| + | 2| C| + +---+-----+ - Parameters - ---------- - funcName : str - function name that follows the SQL identifier syntax (can be quoted, can be qualified) - cols : :class:`~pyspark.sql.Column` or str - column names or :class:`~pyspark.sql.Column`\\s to be used in the function + Example 2: Use `desc` in `orderBy` function to sort the DataFrame. - Returns - ------- - :class:`~pyspark.sql.Column` - result of executed function. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.orderBy(sf.desc("value")).show() + +---+-----+ + | id|value| + +---+-----+ + | 2| C| + | 4| B| + | 3| A| + +---+-----+ - Examples - -------- - >>> from pyspark.sql.functions import call_udf, col - >>> from pyspark.sql.types import IntegerType, StringType - >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) - >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) - >>> df.select(call_function("intX2", "id")).show() - +---------+ - |intX2(id)| - +---------+ - | 2| - | 4| - | 6| - +---------+ - >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) - >>> df.select(call_function("strX2", col("name"))).show() - +-----------+ - |strX2(name)| - +-----------+ - | aa| - | bb| - | cc| - +-----------+ - >>> df.select(call_function("avg", col("id"))).show() - +-------+ - |avg(id)| - +-------+ - | 2.0| - +-------+ - >>> _ = spark.sql("CREATE FUNCTION custom_avg AS 'test.org.apache.spark.sql.MyDoubleAvg'") - ... # doctest: +SKIP - >>> df.select(call_function("custom_avg", col("id"))).show() - ... # doctest: +SKIP - +------------------------------------+ - |spark_catalog.default.custom_avg(id)| - +------------------------------------+ - | 102.0| - +------------------------------------+ - >>> df.select(call_function("spark_catalog.default.custom_avg", col("id"))).show() - ... # doctest: +SKIP - +------------------------------------+ - |spark_catalog.default.custom_avg(id)| - +------------------------------------+ - | 102.0| - +------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + Example 3: Combine `asc` with `desc` to sort by multiple columns. - sc = _get_active_spark_context() - return _invoke_function("call_function", funcName, _to_seq(sc, cols, _to_java_column)) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], + ... ['id', 'group', 'value']) + >>> df.sort(sf.desc("group"), sf.asc("value")).show() + +---+-----+-----+ + | id|group|value| + +---+-----+-----+ + | 1| B| 3| + | 3| A| 2| + | 2| A| 4| + +---+-----+-----+ + Example 4: Implement `desc` from column expression. -# ---------------------- Conditional Functions ---------------------- + >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) + >>> df.sort(df.id.desc()).show() + +---+-----+ + | id|value| + +---+-----+ + | 4| B| + | 3| A| + | 2| C| + +---+-----+ + """ + return col.desc() if isinstance(col, Column) else _invoke_function("desc", col) @_try_remote_functions -def coalesce(*cols: "ColumnOrName") -> Column: - """Returns the first column that is not null. +def sqrt(col: "ColumnOrName") -> Column: + """ + Computes the square root of the specified float value. - .. versionadded:: 1.4.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - list of columns to work on. - Each a column of any type. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - value of the first column that is not null. - Returns a column of the same type as the input. + column for computed results. + Returns a column that evaluates to a double. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None, None), (1, None), (None, 2)], ("a", "b")) - >>> df.show() - +----+----+ - | a| b| - +----+----+ - |NULL|NULL| - | 1|NULL| - |NULL| 2| - +----+----+ - - >>> df.select('*', sf.coalesce("a", df["b"])).show() - +----+----+--------------+ - | a| b|coalesce(a, b)| - +----+----+--------------+ - |NULL|NULL| NULL| - | 1|NULL| 1| - |NULL| 2| 2| - +----+----+--------------+ - - >>> df.select('*', sf.coalesce(df["a"], lit(0.0))).show() - +----+----+----------------+ - | a| b|coalesce(a, 0.0)| - +----+----+----------------+ - |NULL|NULL| 0.0| - | 1|NULL| 1.0| - |NULL| 2| 0.0| - +----+----+----------------+ + >>> spark.sql( + ... "SELECT * FROM VALUES (-1), (0), (1), (4), (NULL) AS TAB(value)" + ... ).select("*", sf.sqrt("value")).show() + +-----+-----------+ + |value|SQRT(value)| + +-----+-----------+ + | -1| NaN| + | 0| 0.0| + | 1| 1.0| + | 4| 2.0| + | NULL| NULL| + +-----+-----------+ """ - return _invoke_function_over_seq_of_columns("coalesce", cols) + return _invoke_function_over_columns("sqrt", col) @_try_remote_functions -def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns col1 if it is not NaN, or col2 if col1 is NaN. - - Both inputs should be floating point columns (:class:`DoubleType` or :class:`FloatType`). - - .. versionadded:: 1.6.0 +def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """ + Returns the sum of `left`and `right` and the result is null on overflow. + The acceptable input types are the same with the `+` operator. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to check. - A column that evaluates to a double or float. - col2 : :class:`~pyspark.sql.Column` or column name - second column to return if first is NaN. - A column that evaluates to a double or float. - - Returns - ------- - :class:`~pyspark.sql.Column` - value from first column or second if first is NaN . - Returns a column of the same type as the first input. + left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. + right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) - >>> df.select("*", sf.nanvl("a", "b"), sf.nanvl(df.a, df.b)).show() - +---+---+-----------+-----------+ - | a| b|nanvl(a, b)|nanvl(a, b)| - +---+---+-----------+-----------+ - |1.0|NaN| 1.0| 1.0| - |NaN|2.0| 2.0| 2.0| - +---+---+-----------+-----------+ - """ - return _invoke_function_over_columns("nanvl", col1, col2) - + Example 1: Integer plus Integer. -@_try_remote_functions -def when(condition: Column, value: Any) -> Column: - """Evaluates a list of conditions and returns one of multiple possible result expressions. - If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched - conditions. + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(1982, 15), (1990, 2)], ["birth", "age"] + ... ).select("*", sf.try_add("birth", "age")).show() + +-----+---+-------------------+ + |birth|age|try_add(birth, age)| + +-----+---+-------------------+ + | 1982| 15| 1997| + | 1990| 2| 1992| + +-----+---+-------------------+ - .. versionadded:: 1.4.0 + Example 2: Date plus Integer. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-09-30')) AS TAB(date)" + ... ).select("*", sf.try_add("date", sf.lit(1))).show() + +----------+----------------+ + | date|try_add(date, 1)| + +----------+----------------+ + |2015-09-30| 2015-10-01| + +----------+----------------+ - Parameters - ---------- - condition : :class:`~pyspark.sql.Column` - a boolean :class:`~pyspark.sql.Column` expression. - A column that evaluates to a boolean. - value : - a literal value, or a :class:`~pyspark.sql.Column` expression. - A column of any type. + Example 3: Date plus Interval. - Returns - ------- - :class:`~pyspark.sql.Column` - column representing when expression. - Returns a column of the same type as the input. + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" + ... ).select("*", sf.try_add("date", "itvl")).show() + +----------+-----------------+-------------------+ + | date| itvl|try_add(date, itvl)| + +----------+-----------------+-------------------+ + |2015-09-30|INTERVAL '1' YEAR| 2016-09-30| + +----------+-----------------+-------------------+ - See Also - -------- - :meth:`pyspark.sql.Column.when` - :meth:`pyspark.sql.Column.otherwise` + Example 4: Interval plus Interval. - Examples - -------- >>> import pyspark.sql.functions as sf - >>> df = spark.range(3) - >>> df.select("*", sf.when(df['id'] == 2, 3).otherwise(4)).show() - +---+------------------------------------+ - | id|CASE WHEN (id = 2) THEN 3 ELSE 4 END| - +---+------------------------------------+ - | 0| 4| - | 1| 4| - | 2| 3| - +---+------------------------------------+ + >>> spark.sql( + ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" + ... ).select("*", sf.try_add("itvl1", "itvl2")).show() + +-----------------+-----------------+---------------------+ + | itvl1| itvl2|try_add(itvl1, itvl2)| + +-----------------+-----------------+---------------------+ + |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '3' YEAR| + +-----------------+-----------------+---------------------+ - >>> df.select("*", sf.when(df.id == 2, df.id + 1)).show() - +---+------------------------------------+ - | id|CASE WHEN (id = 2) THEN (id + 1) END| - +---+------------------------------------+ - | 0| NULL| - | 1| NULL| - | 2| 3| - +---+------------------------------------+ - """ - # Explicitly not using ColumnOrName type here to make reading condition less opaque - if not isinstance(condition, Column): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column", - "arg_name": "condition", - "arg_type": type(condition).__name__, - }, - ) - value = _enum_to_value(value) - v = value._jc if isinstance(value, Column) else _enum_to_value(value) + Example 5: Overflow results in NULL when ANSI mode is on - return _invoke_function("when", condition._jc, v) + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_add(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-------------------------------------------------+ + |try_add(9223372036854775807, 9223372036854775807)| + +-------------------------------------------------+ + | NULL| + +-------------------------------------------------+ + """ + return _invoke_function_over_columns("try_add", left, right) @_try_remote_functions -def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def try_avg(col: "ColumnOrName") -> Column: """ - Returns `col2` if `col1` is null, or `col1` otherwise. + Returns the mean calculated from values of a group and the result is null on overflow. .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - col2 : :class:`~pyspark.sql.Column` or str + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- + Example 1: Calculating the average age + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) - >>> df.select(sf.ifnull(df.e, sf.lit(8))).show() + >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) + >>> df.select(sf.try_avg("age")).show() +------------+ - |ifnull(e, 8)| + |try_avg(age)| +------------+ - | 8| - | 1| + | 8.5| +------------+ + + Example 2: Calculating the average age with None + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.try_avg("age")).show() + +------------+ + |try_avg(age)| + +------------+ + | 3.0| + +------------+ + + Example 3: Overflow results in NULL when ANSI mode is on + + >>> from decimal import Decimal + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.createDataFrame( + ... [(Decimal("1" * 38),), (Decimal(0),)], "number DECIMAL(38, 0)") + ... df.select(sf.try_avg(df.number)).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +---------------+ + |try_avg(number)| + +---------------+ + | NULL| + +---------------+ """ - return _invoke_function_over_columns("ifnull", col1, col2) + return _invoke_function_over_columns("try_avg", col) @_try_remote_functions -def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns null if `col1` equals to `col2`, or `col1` otherwise. + Returns `dividend`/`divisor`. It always performs floating point division. Its result is + always null if `divisor` is 0. .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. - col2 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. + left : :class:`~pyspark.sql.Column` or column name + dividend. + A column that evaluates to a numeric or interval. + right : :class:`~pyspark.sql.Column` or column name + divisor. + A column that evaluates to a numeric. Examples -------- + Example 1: Integer divided by Integer. + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) - >>> df.select('*', sf.nullif(df.a, df.b)).show() - +----+----+------------+ - | a| b|nullif(a, b)| - +----+----+------------+ - |NULL|NULL| NULL| - | 1| 9| 1| - +----+----+------------+ + >>> spark.createDataFrame( + ... [(6000, 15), (1990, 2), (1234, 0)], ["a", "b"] + ... ).select("*", sf.try_divide("a", "b")).show() + +----+---+----------------+ + | a| b|try_divide(a, b)| + +----+---+----------------+ + |6000| 15| 400.0| + |1990| 2| 995.0| + |1234| 0| NULL| + +----+---+----------------+ - >>> df.select('*', sf.nullif('a', 'b')).show() - +----+----+------------+ - | a| b|nullif(a, b)| - +----+----+------------+ - |NULL|NULL| NULL| - | 1| 9| 1| - +----+----+------------+ + Example 2: Interval divided by Integer. + + >>> import pyspark.sql.functions as sf + >>> df = spark.range(4).select(sf.make_interval(sf.lit(1)).alias("itvl"), "id") + >>> df.select("*", sf.try_divide("itvl", "id")).show() + +-------+---+--------------------+ + | itvl| id|try_divide(itvl, id)| + +-------+---+--------------------+ + |1 years| 0| NULL| + |1 years| 1| 1 years| + |1 years| 2| 6 months| + |1 years| 3| 4 months| + +-------+---+--------------------+ + + Example 3: Exception during division, resulting in NULL when ANSI mode is on + + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_divide("id", sf.lit(0))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-----------------+ + |try_divide(id, 0)| + +-----------------+ + | NULL| + +-----------------+ """ - return _invoke_function_over_columns("nullif", col1, col2) + return _invoke_function_over_columns("try_divide", left, right) @_try_remote_functions -def nullifzero(col: "ColumnOrName") -> Column: +def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns null if `col` is equal to zero, or `col` otherwise. + Returns the remainder after `dividend`/`divisor`. Its result is + always null if `divisor` is 0. .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + left : :class:`~pyspark.sql.Column` or column name + dividend. + A column that evaluates to a numeric. + right : :class:`~pyspark.sql.Column` or column name + divisor. A column that evaluates to a numeric. Examples -------- + Example 1: Integer divided by Integer. + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(0,), (1,)], ["a"]) - >>> df.select('*', sf.nullifzero(df.a)).show() - +---+-------------+ - | a|nullifzero(a)| - +---+-------------+ - | 0| NULL| - | 1| 1| - +---+-------------+ + >>> spark.createDataFrame( + ... [(6000, 15), (3, 2), (1234, 0)], ["a", "b"] + ... ).select("*", sf.try_mod("a", "b")).show() + +----+---+-------------+ + | a| b|try_mod(a, b)| + +----+---+-------------+ + |6000| 15| 0| + | 3| 2| 1| + |1234| 0| NULL| + +----+---+-------------+ - >>> df.select('*', sf.nullifzero('a')).show() - +---+-------------+ - | a|nullifzero(a)| - +---+-------------+ - | 0| NULL| - | 1| 1| - +---+-------------+ + Example 2: Exception during division, resulting in NULL when ANSI mode is on + + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_mod("id", sf.lit(0))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +--------------+ + |try_mod(id, 0)| + +--------------+ + | NULL| + +--------------+ """ - return _invoke_function_over_columns("nullifzero", col) + return _invoke_function_over_columns("try_mod", left, right) @_try_remote_functions -def nvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns `col2` if `col1` is null, or `col1` otherwise. + Returns `left`*`right` and the result is null on overflow. The acceptable input types are the + same with the `*` operator. .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - col2 : :class:`~pyspark.sql.Column` or column name - - See Also - -------- - :meth:`pyspark.sql.functions.nvl2` + left : :class:`~pyspark.sql.Column` or column name + multiplicand. + A column that evaluates to a numeric or interval. + right : :class:`~pyspark.sql.Column` or column name + multiplier. + A column that evaluates to a numeric or interval. Examples -------- + Example 1: Integer multiplied by Integer. + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None, 8,), (1, 9,)], ["a", "b"]) - >>> df.select('*', sf.nvl(df.a, df.b)).show() - +----+---+---------+ - | a| b|nvl(a, b)| - +----+---+---------+ - |NULL| 8| 8| - | 1| 9| 1| - +----+---+---------+ + >>> spark.createDataFrame( + ... [(6000, 15), (1990, 2)], ["a", "b"] + ... ).select("*", sf.try_multiply("a", "b")).show() + +----+---+------------------+ + | a| b|try_multiply(a, b)| + +----+---+------------------+ + |6000| 15| 90000| + |1990| 2| 3980| + +----+---+------------------+ - >>> df.select('*', sf.nvl('a', 'b')).show() - +----+---+---------+ - | a| b|nvl(a, b)| - +----+---+---------+ - |NULL| 8| 8| - | 1| 9| 1| - +----+---+---------+ + Example 2: Interval multiplied by Integer. + + >>> import pyspark.sql.functions as sf + >>> df = spark.range(6).select(sf.make_interval(sf.col("id"), sf.lit(3)).alias("itvl"), "id") + >>> df.select("*", sf.try_multiply("itvl", "id")).show() + +----------------+---+----------------------+ + | itvl| id|try_multiply(itvl, id)| + +----------------+---+----------------------+ + | 3 months| 0| 0 seconds| + |1 years 3 months| 1| 1 years 3 months| + |2 years 3 months| 2| 4 years 6 months| + |3 years 3 months| 3| 9 years 9 months| + |4 years 3 months| 4| 17 years| + |5 years 3 months| 5| 26 years 3 months| + +----------------+---+----------------------+ + + Example 3: Overflow results in NULL when ANSI mode is on + + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_multiply(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +------------------------------------------------------+ + |try_multiply(9223372036854775807, 9223372036854775807)| + +------------------------------------------------------+ + | NULL| + +------------------------------------------------------+ """ - return _invoke_function_over_columns("nvl", col1, col2) + return _invoke_function_over_columns("try_multiply", left, right) @_try_remote_functions -def nvl2(col1: "ColumnOrName", col2: "ColumnOrName", col3: "ColumnOrName") -> Column: +def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns `col2` if `col1` is not null, or `col3` otherwise. + Returns `left`-`right` and the result is null on overflow. The acceptable input types are the + same with the `-` operator. .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - col2 : :class:`~pyspark.sql.Column` or column name - col3 : :class:`~pyspark.sql.Column` or column name - - See Also - -------- - :meth:`pyspark.sql.functions.nvl` + left : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. + right : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric, interval, date, timestamp, or time. Examples -------- + Example 1: Integer minus Integer. + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None, 8, 6,), (1, 9, 9,)], ["a", "b", "c"]) - >>> df.select('*', sf.nvl2(df.a, df.b, df.c)).show() - +----+---+---+-------------+ - | a| b| c|nvl2(a, b, c)| - +----+---+---+-------------+ - |NULL| 8| 6| 6| - | 1| 9| 9| 9| - +----+---+---+-------------+ + >>> spark.createDataFrame( + ... [(1982, 15), (1990, 2)], ["birth", "age"] + ... ).select("*", sf.try_subtract("birth", "age")).show() + +-----+---+------------------------+ + |birth|age|try_subtract(birth, age)| + +-----+---+------------------------+ + | 1982| 15| 1967| + | 1990| 2| 1988| + +-----+---+------------------------+ - >>> df.select('*', sf.nvl2('a', 'b', 'c')).show() - +----+---+---+-------------+ - | a| b| c|nvl2(a, b, c)| - +----+---+---+-------------+ - |NULL| 8| 6| 6| - | 1| 9| 9| 9| - +----+---+---+-------------+ + Example 2: Date minus Integer. + + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-10-01')) AS TAB(date)" + ... ).select("*", sf.try_subtract("date", sf.lit(1))).show() + +----------+---------------------+ + | date|try_subtract(date, 1)| + +----------+---------------------+ + |2015-10-01| 2015-09-30| + +----------+---------------------+ + + Example 3: Date minus Interval. + + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" + ... ).select("*", sf.try_subtract("date", "itvl")).show() + +----------+-----------------+------------------------+ + | date| itvl|try_subtract(date, itvl)| + +----------+-----------------+------------------------+ + |2015-09-30|INTERVAL '1' YEAR| 2014-09-30| + +----------+-----------------+------------------------+ + + Example 4: Interval minus Interval. + + >>> import pyspark.sql.functions as sf + >>> spark.sql( + ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" + ... ).select("*", sf.try_subtract("itvl1", "itvl2")).show() + +-----------------+-----------------+--------------------------+ + | itvl1| itvl2|try_subtract(itvl1, itvl2)| + +-----------------+-----------------+--------------------------+ + |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '-1' YEAR| + +-----------------+-----------------+--------------------------+ + + Example 5: Overflow results in NULL when ANSI mode is on + + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... spark.range(1).select(sf.try_subtract(sf.lit(-sys.maxsize), sf.lit(sys.maxsize))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-------------------------------------------------------+ + |try_subtract(-9223372036854775807, 9223372036854775807)| + +-------------------------------------------------------+ + | NULL| + +-------------------------------------------------------+ """ - return _invoke_function_over_columns("nvl2", col1, col2, col3) + return _invoke_function_over_columns("try_subtract", left, right) @_try_remote_functions -def zeroifnull(col: "ColumnOrName") -> Column: +def try_sum(col: "ColumnOrName") -> Column: """ - Returns zero if `col` is null, or `col` otherwise. + Returns the sum calculated from values of a group and the result is null on overflow. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a numeric or interval. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(None,), (1,)], ["a"]) - >>> df.select('*', sf.zeroifnull(df.a)).show() - +----+-------------+ - | a|zeroifnull(a)| - +----+-------------+ - |NULL| 0| - | 1| 1| - +----+-------------+ + Example 1: Calculating the sum of values in a column - >>> df.select('*', sf.zeroifnull('a')).show() - +----+-------------+ - | a|zeroifnull(a)| - +----+-------------+ - |NULL| 0| - | 1| 1| - +----+-------------+ - """ - return _invoke_function_over_columns("zeroifnull", col) + >>> from pyspark.sql import functions as sf + >>> spark.range(10).select(sf.try_sum("id")).show() + +-----------+ + |try_sum(id)| + +-----------+ + | 45| + +-----------+ + + Example 2: Using a plus expression together to calculate the sum + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) + >>> df.select(sf.try_sum(sf.col("A") + sf.col("B"))).show() + +----------------+ + |try_sum((A + B))| + +----------------+ + | 10| + +----------------+ + + Example 3: Calculating the summation of ages with None + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.try_sum("age")).show() + +------------+ + |try_sum(age)| + +------------+ + | 6| + +------------+ + Example 4: Overflow results in NULL when ANSI mode is on -# ---------------------- Predicate Functions ---------------------- + >>> from decimal import Decimal + >>> import pyspark.sql.functions as sf + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.createDataFrame([(Decimal("1" * 38),)] * 10, "number DECIMAL(38, 0)") + ... df.select(sf.try_sum(df.number)).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +---------------+ + |try_sum(number)| + +---------------+ + | NULL| + +---------------+ + """ + return _invoke_function_over_columns("try_sum", col) @_try_remote_functions -def isnan(col: "ColumnOrName") -> Column: - """An expression that returns true if the column is NaN. +def abs(col: "ColumnOrName") -> Column: + """ + Mathematical Function: Computes the absolute value of the given column or expression. - .. versionadded:: 1.6.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -890,554 +1051,556 @@ def isnan(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a double or float. + The target column or expression to compute the absolute value on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` - True if value is NaN and False otherwise. - Returns a column that evaluates to a boolean. - - See Also - -------- - :meth:`pyspark.sql.functions.isnull` - :meth:`pyspark.sql.functions.isnotnull` + A new column object representing the absolute value of the input. + Returns a column of the same type as the input. Examples -------- + Example 1: Compute the absolute value of a long column + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) - >>> df.select("*", sf.isnan("a"), sf.isnan(df.b)).show() - +---+---+--------+--------+ - | a| b|isnan(a)|isnan(b)| - +---+---+--------+--------+ - |1.0|NaN| false| true| - |NaN|2.0| true| false| - +---+---+--------+--------+ + >>> df = spark.createDataFrame([(-1,), (-2,), (-3,), (None,)], ["value"]) + >>> df.select("*", sf.abs(df.value)).show() + +-----+----------+ + |value|abs(value)| + +-----+----------+ + | -1| 1| + | -2| 2| + | -3| 3| + | NULL| NULL| + +-----+----------+ + + Example 2: Compute the absolute value of a double column + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(-1.5,), (-2.5,), (None,), (float("nan"),)], ["value"]) + >>> df.select("*", sf.abs(df.value)).show() + +-----+----------+ + |value|abs(value)| + +-----+----------+ + | -1.5| 1.5| + | -2.5| 2.5| + | NULL| NULL| + | NaN| NaN| + +-----+----------+ + + Example 3: Compute the absolute value of an expression + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 1), (2, -2), (3, 3)], ["id", "value"]) + >>> df.select("*", sf.abs(df.id - df.value)).show() + +---+-----+-----------------+ + | id|value|abs((id - value))| + +---+-----+-----------------+ + | 1| 1| 0| + | 2| -2| 4| + | 3| 3| 0| + +---+-----+-----------------+ """ - return _invoke_function_over_columns("isnan", col) + return _invoke_function_over_columns("abs", col) @_try_remote_functions -def isnull(col: "ColumnOrName") -> Column: - """An expression that returns true if the column is null. +def mode(col: "ColumnOrName", deterministic: bool = False) -> Column: + """ + Returns the most frequent value in a group. - .. versionadded:: 1.6.0 + .. versionadded:: 3.4.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionchanged:: 4.0.0 + Supports deterministic argument. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. A column of any type. + deterministic : bool, optional + if there are multiple equally-frequent results then return the lowest (defaults to false). + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - True if value is null and False otherwise. - Returns a column that evaluates to a boolean. + the most frequent value in a group. - See Also - -------- - :meth:`pyspark.sql.functions.isnan` - :meth:`pyspark.sql.functions.isnotnull` + Notes + ----- + Supports Spark Connect. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, None), (None, 2)], ("a", "b")) - >>> df.select("*", sf.isnull("a"), isnull(df.b)).show() - +----+----+-----------+-----------+ - | a| b|(a IS NULL)|(b IS NULL)| - +----+----+-----------+-----------+ - | 1|NULL| false| true| - |NULL| 2| true| false| - +----+----+-----------+-----------+ + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.mode("year")).sort("course").show() + +------+----------+ + |course|mode(year)| + +------+----------+ + | Java| 2012| + |dotNET| 2012| + +------+----------+ + + When multiple values have the same greatest frequency then either any of values is returned if + deterministic is false or is not defined, or the lowest value is returned if deterministic is + true. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(-10,), (0,), (10,)], ["col"]) + >>> df.select(sf.mode("col", False)).show() # doctest: +SKIP + +---------+ + |mode(col)| + +---------+ + | 0| + +---------+ + + >>> df.select(sf.mode("col", True)).show() + +---------------------------------------+ + |mode() WITHIN GROUP (ORDER BY col DESC)| + +---------------------------------------+ + | -10| + +---------------------------------------+ """ - return _invoke_function_over_columns("isnull", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("mode", _to_java_column(col), _enum_to_value(deterministic)) @_try_remote_functions -def rlike(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. +def max(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns the maximum value of the expression in a group. - .. versionadded:: 3.5.0 + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The target column on which the maximum value is computed. Returns ------- :class:`~pyspark.sql.Column` - true if `str` matches a Java regex, or false otherwise. - Returns a column that evaluates to a boolean. + A column that contains the maximum value computed. See Also -------- - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.regexp_like` - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.ilike` + :meth:`pyspark.sql.functions.min` + :meth:`pyspark.sql.functions.avg` + :meth:`pyspark.sql.functions.sum` + + Notes + ----- + - Null values are ignored during the computation. + - NaN values are larger than any other numeric value. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"(\d+)")], ["str", "regexp"]) - >>> df.select('*', sf.rlike('str', sf.lit(r'(\d+)'))).show() - +---------+------+-----------------+ - | str|regexp|RLIKE(str, (\d+))| - +---------+------+-----------------+ - |1a 2b 14m| (\d+)| true| - +---------+------+-----------------+ - - >>> df.select('*', sf.rlike('str', sf.lit(r'\d{2}b'))).show() - +---------+------+------------------+ - | str|regexp|RLIKE(str, \d{2}b)| - +---------+------+------------------+ - |1a 2b 14m| (\d+)| false| - +---------+------+------------------+ + Example 1: Compute the maximum value of a numeric column - >>> df.select('*', sf.rlike("str", sf.col("regexp"))).show() - +---------+------+------------------+ - | str|regexp|RLIKE(str, regexp)| - +---------+------+------------------+ - |1a 2b 14m| (\d+)| true| - +---------+------+------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.range(10) + >>> df.select(sf.max(df.id)).show() + +-------+ + |max(id)| + +-------+ + | 9| + +-------+ - >>> df.select('*', sf.rlike("str", "regexp")).show() - +---------+------+------------------+ - | str|regexp|RLIKE(str, regexp)| - +---------+------+------------------+ - |1a 2b 14m| (\d+)| true| - +---------+------+------------------+ - """ - return _invoke_function_over_columns("rlike", str, regexp) + Example 2: Compute the maximum value of a string column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("A",), ("B",), ("C",)], ["value"]) + >>> df.select(sf.max(df.value)).show() + +----------+ + |max(value)| + +----------+ + | C| + +----------+ -@_try_remote_functions -def regexp(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. + Example 3: Compute the maximum value of a column in a grouped DataFrame - .. versionadded:: 3.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("A", 1), ("A", 2), ("B", 3), ("B", 4)], ["key", "value"]) + >>> df.groupBy("key").agg(sf.max(df.value)).show() + +---+----------+ + |key|max(value)| + +---+----------+ + | A| 2| + | B| 4| + +---+----------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or str - regex pattern to apply. - A column that evaluates to a string. + Example 4: Compute the maximum value of multiple columns in a grouped DataFrame - Returns - ------- - :class:`~pyspark.sql.Column` - true if `str` matches a Java regex, or false otherwise. - Returns a column that evaluates to a boolean. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame( + ... [("A", 1, 2), ("A", 2, 3), ("B", 3, 4), ("B", 4, 5)], ["key", "value1", "value2"]) + >>> df.groupBy("key").agg(sf.max("value1"), sf.max("value2")).show() + +---+-----------+-----------+ + |key|max(value1)|max(value2)| + +---+-----------+-----------+ + | A| 2| 3| + | B| 4| 5| + +---+-----------+-----------+ - See Also - -------- - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp_like` - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.ilike` + Example 5: Compute the maximum value of a column with null values - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp('str', sf.lit(r'(\d+)'))).show() - +------------------+ - |REGEXP(str, (\d+))| - +------------------+ - | true| - +------------------+ + >>> df = spark.createDataFrame([(1,), (2,), (None,)], ["value"]) + >>> df.select(sf.max(df.value)).show() + +----------+ + |max(value)| + +----------+ + | 2| + +----------+ - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp('str', sf.lit(r'\d{2}b'))).show() - +-------------------+ - |REGEXP(str, \d{2}b)| - +-------------------+ - | false| - +-------------------+ + Example 6: Compute the maximum value of a column with "NaN" values >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp('str', sf.col("regexp"))).show() - +-------------------+ - |REGEXP(str, regexp)| - +-------------------+ - | true| - +-------------------+ + >>> df = spark.createDataFrame([(1.1,), (float("nan"),), (3.3,)], ["value"]) + >>> df.select(sf.max(df.value)).show() + +----------+ + |max(value)| + +----------+ + | NaN| + +----------+ """ - return _invoke_function_over_columns("regexp", str, regexp) + return _invoke_function_over_columns("max", col) @_try_remote_functions -def regexp_like(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. +def min(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns the minimum value of the expression in a group. - .. versionadded:: 3.5.0 + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or str - regex pattern to apply. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The target column on which the minimum value is computed. Returns ------- :class:`~pyspark.sql.Column` - true if `str` matches a Java regex, or false otherwise. - Returns a column that evaluates to a boolean. + A column that contains the minimum value computed. See Also -------- - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.ilike` + :meth:`pyspark.sql.functions.max` + :meth:`pyspark.sql.functions.avg` + :meth:`pyspark.sql.functions.sum` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp_like('str', sf.lit(r'(\d+)'))).show() - +-----------------------+ - |REGEXP_LIKE(str, (\d+))| - +-----------------------+ - | true| - +-----------------------+ + Example 1: Compute the minimum value of a numeric column >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp_like('str', sf.lit(r'\d{2}b'))).show() - +------------------------+ - |REGEXP_LIKE(str, \d{2}b)| - +------------------------+ - | false| - +------------------------+ + >>> df = spark.range(10) + >>> df.select(sf.min(df.id)).show() + +-------+ + |min(id)| + +-------+ + | 0| + +-------+ - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] - ... ).select(sf.regexp_like('str', sf.col("regexp"))).show() - +------------------------+ - |REGEXP_LIKE(str, regexp)| - +------------------------+ - | true| - +------------------------+ - """ - return _invoke_function_over_columns("regexp_like", str, regexp) + Example 2: Compute the minimum value of a string column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Alice",), ("Bob",), ("Charlie",)], ["name"]) + >>> df.select(sf.min("name")).show() + +---------+ + |min(name)| + +---------+ + | Alice| + +---------+ -@_try_remote_functions -def like( - str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None -) -> Column: - """ - Returns true if str matches `pattern` with `escape`, - null if any arguments are null, false otherwise. - The default escape character is the '\'. + Example 3: Compute the minimum value of a column with null values - .. versionadded:: 3.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1,), (None,), (3,)], ["value"]) + >>> df.select(sf.min("value")).show() + +----------+ + |min(value)| + +----------+ + | 1| + +----------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - A string. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or str - A string. The pattern is a string which is matched literally, with - exception to the following special symbols: - _ matches any one character in the input (similar to . in posix regular expressions) - % matches zero or more characters in the input (similar to .* in posix regular - expressions) - Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order - to match "\abc", the pattern should be "\\abc". - When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back - to Spark 1.6 behavior regarding string literal parsing. For example, if the config is - enabled, the pattern to match "\abc" should be "\abc". - A column that evaluates to a string. - escapeChar : :class:`~pyspark.sql.Column`, optional - An character added since Spark 3.0. The default escape character is the '\'. - If an escape character precedes a special symbol or another escape character, the - following character is matched literally. It is invalid to escape any other character. - A column that evaluates to a string. + Example 4: Compute the minimum value of a column in a grouped DataFrame - See Also - -------- - :meth:`pyspark.sql.functions.ilike` - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.regexp_like` + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Alice", 1), ("Alice", 2), ("Bob", 3)], ["name", "value"]) + >>> df.groupBy("name").agg(sf.min("value")).show() + +-----+----------+ + | name|min(value)| + +-----+----------+ + |Alice| 1| + | Bob| 3| + +-----+----------+ - Examples - -------- - >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) - >>> df.select(like(df.a, df.b).alias('r')).collect() - [Row(r=True)] + Example 5: Compute the minimum value of a column in a DataFrame with multiple columns + >>> import pyspark.sql.functions as sf >>> df = spark.createDataFrame( - ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], - ... ['a', 'b'] - ... ) - >>> df.select(like(df.a, df.b, lit('/')).alias('r')).collect() - [Row(r=True)] + ... [("Alice", 1, 100), ("Bob", 2, 200), ("Charlie", 3, 300)], + ... ["name", "value1", "value2"]) + >>> df.select(sf.min("value1"), sf.min("value2")).show() + +-----------+-----------+ + |min(value1)|min(value2)| + +-----------+-----------+ + | 1| 100| + +-----------+-----------+ """ - if escapeChar is not None: - return _invoke_function_over_columns("like", str, pattern, escapeChar) - else: - return _invoke_function_over_columns("like", str, pattern) + return _invoke_function_over_columns("min", col) @_try_remote_functions -def ilike( - str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None -) -> Column: +def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: """ - Returns true if str matches `pattern` with `escape` case-insensitively, - null if any arguments are null, false otherwise. - The default escape character is the '\'. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - A string. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or str - A string. The pattern is a string which is matched literally, with - exception to the following special symbols: - _ matches any one character in the input (similar to . in posix regular expressions) - % matches zero or more characters in the input (similar to .* in posix regular - expressions) - Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order - to match "\abc", the pattern should be "\\abc". - When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back - to Spark 1.6 behavior regarding string literal parsing. For example, if the config is - enabled, the pattern to match "\abc" should be "\abc". - A column that evaluates to a string. - escapeChar : :class:`~pyspark.sql.Column`, optional - An character added since Spark 3.0. The default escape character is the '\'. - If an escape character precedes a special symbol or another escape character, the - following character is matched literally. It is invalid to escape any other character. - A column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.like` - :meth:`pyspark.sql.functions.rlike` - :meth:`pyspark.sql.functions.regexp` - :meth:`pyspark.sql.functions.regexp_like` + Returns the value(s) from the `col` parameter that are associated with the maximum value(s) + from the `ord` parameter. This function is often used to find the `col` parameter value + corresponding to the maximum `ord` parameter value within each group when used with groupBy(). - Examples - -------- - >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) - >>> df.select(ilike(df.a, df.b).alias('r')).collect() - [Row(r=True)] + When `k` is specified, returns an array of up to `k` values associated with the top `k` + maximum values from `ord`. - >>> df = spark.createDataFrame( - ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], - ... ['a', 'b'] - ... ) - >>> df.select(ilike(df.a, df.b, lit('/')).alias('r')).collect() - [Row(r=True)] - """ - if escapeChar is not None: - return _invoke_function_over_columns("ilike", str, pattern, escapeChar) - else: - return _invoke_function_over_columns("ilike", str, pattern) + .. versionadded:: 3.3.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. -@_try_remote_functions -def isnotnull(col: "ColumnOrName") -> Column: - """ - Returns true if `col` is not null, or false otherwise. + .. versionchanged:: 4.2.0 + Added optional `k` parameter to return top-k values. - .. versionadded:: 3.5.0 + Notes + ----- + The function is non-deterministic so the output order can be different for those + associated the same values of `col`. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name + The column representing the values to be returned. This could be the column instance + or the column name as string. + A column of any type. + ord : :class:`~pyspark.sql.Column` or column name + The column that needs to be maximized. This could be the column instance + or the column name as string. + A column of any orderable type. + k : int, optional + If specified, returns an array of up to `k` values associated with the top `k` + maximum ordering values, sorted in descending order by the ordering column. + Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. - See Also - -------- - :meth:`pyspark.sql.functions.isnan` - :meth:`pyspark.sql.functions.isnull` + Returns + ------- + :class:`~pyspark.sql.Column` + A column object representing the value from `col` that is associated with + the maximum value from `ord`. If `k` is specified, returns an array of values. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) - >>> df.select('*', sf.isnotnull(df.e)).show() - +----+---------------+ - | e|(e IS NOT NULL)| - +----+---------------+ - |NULL| false| - | 1| true| - +----+---------------+ + Example 1: Using `max_by` with groupBy - >>> df.select('*', sf.isnotnull('e')).show() - +----+---------------+ - | e|(e IS NOT NULL)| - +----+---------------+ - |NULL| false| - | 1| true| - +----+---------------+ - """ - return _invoke_function_over_columns("isnotnull", col) + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.max_by("year", "earnings")).sort("course").show() + +------+----------------------+ + |course|max_by(year, earnings)| + +------+----------------------+ + | Java| 2013| + |dotNET| 2013| + +------+----------------------+ + Example 2: Using `max_by` with different data types -@_try_remote_functions -def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """ - Returns same result as the EQUAL(=) operator for non-null operands, - but returns true if both are null, false if one of them is null. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), + ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.max_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|max_by(name, years_in_dept)| + +----------+---------------------------+ + | IT| Charlie| + | Marketing| Anna| + +----------+---------------------------+ - .. versionadded:: 3.5.0 + Example 3: Using `max_by` where `ord` has multiple maximum values - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. - col2 : :class:`~pyspark.sql.Column` or column name - A column of any orderable type. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), + ... ("Finance", "George", 9), ("Consult", "Henry", 7)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.max_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|max_by(name, years_in_dept)| + +----------+---------------------------+ + | Consult| Henry| + | Finance| George| + +----------+---------------------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) - >>> df.select('*', sf.equal_null(df.a, df.b)).show() - +----+----+----------------+ - | a| b|equal_null(a, b)| - +----+----+----------------+ - |NULL|NULL| true| - | 1| 9| false| - +----+----+----------------+ + Example 4: Using `max_by` with `k` to get top-k values - >>> df.select('*', sf.equal_null('a', 'b')).show() - +----+----+----------------+ - | a| b|equal_null(a, b)| - +----+----+----------------+ - |NULL|NULL| true| - | 1| 9| false| - +----+----+----------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], + ... schema=("x", "y")) + >>> df.select(sf.max_by("x", "y", 2)).show() + +---------------+ + |max_by(x, y, 2)| + +---------------+ + | [b, d]| + +---------------+ """ - return _invoke_function_over_columns("equal_null", col1, col2) - - -# ---------------------- Sort Functions ---------------------- + if k is not None: + return _invoke_function_over_columns("max_by", col, ord, lit(k)) + return _invoke_function_over_columns("max_by", col, ord) @_try_remote_functions -def asc(col: "ColumnOrName") -> Column: +def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: """ - Returns a sort expression for the target column in ascending order. - This function is used in `sort` and `orderBy` functions. + Returns the value(s) from the `col` parameter that are associated with the minimum value(s) + from the `ord` parameter. This function is often used to find the `col` parameter value + corresponding to the minimum `ord` parameter value within each group when used with groupBy(). - .. versionadded:: 1.3.0 + When `k` is specified, returns an array of up to `k` values associated with the bottom `k` + minimum values from `ord`. - .. versionchanged:: 3.4.0 + .. versionadded:: 3.3.0 + + .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.2.0 + Added optional `k` parameter to return bottom-k values. + + Notes + ----- + The function is non-deterministic so the output order can be different for those + associated the same values of `col`. + Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Target column to sort by in the ascending order. + The column representing the values that will be returned. This could be the column instance + or the column name as string. + A column of any type. + ord : :class:`~pyspark.sql.Column` or column name + The column that needs to be minimized. This could be the column instance + or the column name as string. + A column of any orderable type. + k : int, optional + If specified, returns an array of up to `k` values associated with the bottom `k` + minimum ordering values, sorted in ascending order by the ordering column. + Must be a positive integer literal <= 100000. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - The column specifying the sort order. - - See Also - -------- - :meth:`pyspark.sql.functions.asc_nulls_first` - :meth:`pyspark.sql.functions.asc_nulls_last` + Column object that represents the value from `col` associated with + the minimum value from `ord`. If `k` is specified, returns an array of values. Examples -------- - Example 1: Sort DataFrame by 'id' column in ascending order. + Example 1: Using `min_by` with groupBy: - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(sf.asc("id")).show() - +---+-----+ - | id|value| - +---+-----+ - | 2| C| - | 3| A| - | 4| B| - +---+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.min_by("year", "earnings")).sort("course").show() + +------+----------------------+ + |course|min_by(year, earnings)| + +------+----------------------+ + | Java| 2012| + |dotNET| 2012| + +------+----------------------+ - Example 2: Use `asc` in `orderBy` function to sort the DataFrame. + Example 2: Using `min_by` with different data types: - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.orderBy(sf.asc("value")).show() - +---+-----+ - | id|value| - +---+-----+ - | 3| A| - | 4| B| - | 2| C| - +---+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), + ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.min_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|min_by(name, years_in_dept)| + +----------+---------------------------+ + | IT| Bob| + | Marketing| David| + +----------+---------------------------+ - Example 3: Combine `asc` with `desc` to sort by multiple columns. + Example 3: Using `min_by` where `ord` has multiple minimum values: - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], - ... ['id', 'group', 'value']) - >>> df.sort(sf.asc("group"), sf.desc("value")).show() - +---+-----+-----+ - | id|group|value| - +---+-----+-----+ - | 2| A| 4| - | 3| A| 2| - | 1| B| 3| - +---+-----+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), + ... ("Finance", "George", 9), ("Consult", "Henry", 7)], + ... schema=("department", "name", "years_in_dept")) + >>> df.groupby("department").agg( + ... sf.min_by("name", "years_in_dept") + ... ).sort("department").show() + +----------+---------------------------+ + |department|min_by(name, years_in_dept)| + +----------+---------------------------+ + | Consult| Eva| + | Finance| Frank| + +----------+---------------------------+ - Example 4: Implement `asc` from column expression. + Example 4: Using `min_by` with `k` to get bottom-k values - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(df.id.asc()).show() - +---+-----+ - | id|value| - +---+-----+ - | 2| C| - | 3| A| - | 4| B| - +---+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], + ... schema=("x", "y")) + >>> df.select(sf.min_by("x", "y", 2)).show() + +---------------+ + |min_by(x, y, 2)| + +---------------+ + | [a, c]| + +---------------+ """ - return col.asc() if isinstance(col, Column) else _invoke_function("asc", col) + if k is not None: + return _invoke_function_over_columns("min_by", col, ord, lit(k)) + return _invoke_function_over_columns("min_by", col, ord) @_try_remote_functions -def desc(col: "ColumnOrName") -> Column: +def count(col: "ColumnOrName") -> Column: """ - Returns a sort expression for the target column in descending order. - This function is used in `sort` and `orderBy` functions. + Aggregate function: returns the number of items in a group. .. versionadded:: 1.3.0 @@ -1447,83 +1610,71 @@ def desc(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Target column to sort by in the descending order. + target column to compute on. Returns ------- :class:`~pyspark.sql.Column` - The column specifying the sort order. + column for computed results. See Also -------- - :meth:`pyspark.sql.functions.desc_nulls_first` - :meth:`pyspark.sql.functions.desc_nulls_last` + :meth:`pyspark.sql.functions.count_if` Examples -------- - Example 1: Sort DataFrame by 'id' column in descending order. + Example 1: Count all rows in a DataFrame >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(sf.desc("id")).show() - +---+-----+ - | id|value| - +---+-----+ - | 4| B| - | 3| A| - | 2| C| - +---+-----+ + >>> df = spark.createDataFrame([(None,), ("a",), ("b",), ("c",)], schema=["alphabets"]) + >>> df.select(sf.count(sf.expr("*"))).show() + +--------+ + |count(1)| + +--------+ + | 4| + +--------+ - Example 2: Use `desc` in `orderBy` function to sort the DataFrame. + Example 2: Count non-null values in a specific column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.orderBy(sf.desc("value")).show() - +---+-----+ - | id|value| - +---+-----+ - | 2| C| - | 4| B| - | 3| A| - +---+-----+ + >>> df.select(sf.count(df.alphabets)).show() + +----------------+ + |count(alphabets)| + +----------------+ + | 3| + +----------------+ - Example 3: Combine `asc` with `desc` to sort by multiple columns. + Example 3: Count all rows in a DataFrame with multiple columns >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame( - ... [(2, 'A', 4), (1, 'B', 3), (3, 'A', 2)], - ... ['id', 'group', 'value']) - >>> df.sort(sf.desc("group"), sf.asc("value")).show() - +---+-----+-----+ - | id|group|value| - +---+-----+-----+ - | 1| B| 3| - | 3| A| 2| - | 2| A| 4| - +---+-----+-----+ + ... [(1, "apple"), (2, "banana"), (3, None)], schema=["id", "fruit"]) + >>> df.select(sf.count(sf.expr("*"))).show() + +--------+ + |count(1)| + +--------+ + | 3| + +--------+ - Example 4: Implement `desc` from column expression. + Example 4: Count non-null values in multiple columns - >>> df = spark.createDataFrame([(4, 'B'), (3, 'A'), (2, 'C')], ['id', 'value']) - >>> df.sort(df.id.desc()).show() - +---+-----+ - | id|value| - +---+-----+ - | 4| B| - | 3| A| - | 2| C| - +---+-----+ + >>> from pyspark.sql import functions as sf + >>> df.select(sf.count(df.id), sf.count(df.fruit)).show() + +---------+------------+ + |count(id)|count(fruit)| + +---------+------------+ + | 3| 2| + +---------+------------+ """ - return col.desc() if isinstance(col, Column) else _invoke_function("desc", col) + return _invoke_function_over_columns("count", col) @_try_remote_functions -def asc_nulls_first(col: "ColumnOrName") -> Column: +def sum(col: "ColumnOrName") -> Column: """ - Sort Function: Returns a sort expression based on the ascending order of the given - column name, and null values return before non-null values. + Aggregate function: returns the sum of all values in the expression. - .. versionadded:: 2.4.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1531,74 +1682,64 @@ def asc_nulls_first(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the ascending order. + target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. + the column for computed results. See Also -------- - :meth:`pyspark.sql.functions.asc` - :meth:`pyspark.sql.functions.asc_nulls_last` + :meth:`pyspark.sql.functions.min` + :meth:`pyspark.sql.functions.max` + :meth:`pyspark.sql.functions.avg` Examples -------- - Example 1: Sorting a DataFrame with null values in ascending order + Example 1: Calculating the sum of values in a column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_first(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 2|Alice| - | 1| Bob| - +---+-----+ + >>> df = spark.range(10) + >>> df.select(sf.sum(df["id"])).show() + +-------+ + |sum(id)| + +-------+ + | 45| + +-------+ - Example 2: Sorting a DataFrame with multiple columns, null values in ascending order + Example 2: Using a plus expression together to calculate the sum >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.asc_nulls_first(df.name), sf.asc_nulls_first(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 0| NULL| Z| - | 2|Alice| Y| - | 1| Bob| NULL| - +---+-----+-----+ + >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) + >>> df.select(sf.sum(sf.col("A") + sf.col("B"))).show() + +------------+ + |sum((A + B))| + +------------+ + | 10| + +------------+ - Example 3: Sorting a DataFrame with null values in ascending order using column name string + Example 3: Calculating the summation of ages with None - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_first("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 2|Alice| - | 1| Bob| - +---+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.sum("age")).show() + +--------+ + |sum(age)| + +--------+ + | 6| + +--------+ """ - return ( - col.asc_nulls_first() - if isinstance(col, Column) - else _invoke_function("asc_nulls_first", col) - ) + return _invoke_function_over_columns("sum", col) @_try_remote_functions -def asc_nulls_last(col: "ColumnOrName") -> Column: +def avg(col: "ColumnOrName") -> Column: """ - Sort Function: Returns a sort expression based on the ascending order of the given - column name, and null values appear after non-null values. + Aggregate function: returns the average of the values in a group. - .. versionadded:: 2.4.0 + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1606,72 +1747,54 @@ def asc_nulls_last(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the ascending order. + target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. + the column for computed results. See Also -------- - :meth:`pyspark.sql.functions.asc` - :meth:`pyspark.sql.functions.asc_nulls_first` + :meth:`pyspark.sql.functions.min` + :meth:`pyspark.sql.functions.max` + :meth:`pyspark.sql.functions.sum` Examples -------- - Example 1: Sorting a DataFrame with null values in ascending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_last(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 2|Alice| - | 1| Bob| - | 0| NULL| - +---+-----+ - - Example 2: Sorting a DataFrame with multiple columns, null values in ascending order + Example 1: Calculating the average age - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.asc_nulls_last(df.name), sf.asc_nulls_last(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 2|Alice| Y| - | 1| Bob| NULL| - | 0| NULL| Z| - +---+-----+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) + >>> df.select(sf.avg("age")).show() + +--------+ + |avg(age)| + +--------+ + | 8.5| + +--------+ - Example 3: Sorting a DataFrame with null values in ascending order using column name string + Example 2: Calculating the average age with None - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.asc_nulls_last("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 2|Alice| - | 1| Bob| - | 0| NULL| - +---+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.avg("age")).show() + +--------+ + |avg(age)| + +--------+ + | 3.0| + +--------+ """ - return ( - col.asc_nulls_last() if isinstance(col, Column) else _invoke_function("asc_nulls_last", col) - ) + return _invoke_function_over_columns("avg", col) @_try_remote_functions -def desc_nulls_first(col: "ColumnOrName") -> Column: +def mean(col: "ColumnOrName") -> Column: """ - Sort Function: Returns a sort expression based on the descending order of the given - column name, and null values appear before non-null values. + Aggregate function: returns the average of the values in a group. + An alias of :func:`avg`. - .. versionadded:: 2.4.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1679,151 +1802,115 @@ def desc_nulls_first(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the descending order. + target column to compute on. + A column that evaluates to a numeric or interval. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. - - See Also - -------- - :meth:`pyspark.sql.functions.desc` - :meth:`pyspark.sql.functions.desc_nulls_last` + the column for computed results. Examples -------- - Example 1: Sorting a DataFrame with null values in descending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_first(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 1| Bob| - | 2|Alice| - +---+-----+ - - Example 2: Sorting a DataFrame with multiple columns, null values in descending order + Example 1: Calculating the average age - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.desc_nulls_first(df.name), sf.desc_nulls_first(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 0| NULL| Z| - | 1| Bob| NULL| - | 2|Alice| Y| - +---+-----+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) + >>> df.select(sf.mean("age")).show() + +--------+ + |avg(age)| + +--------+ + | 8.5| + +--------+ - Example 3: Sorting a DataFrame with null values in descending order using column name string + Example 2: Calculating the average age with None - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_first("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 0| NULL| - | 1| Bob| - | 2|Alice| - +---+-----+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) + >>> df.select(sf.mean("age")).show() + +--------+ + |avg(age)| + +--------+ + | 3.0| + +--------+ """ - return ( - col.desc_nulls_first() - if isinstance(col, Column) - else _invoke_function("desc_nulls_first", col) - ) + return _invoke_function_over_columns("mean", col) @_try_remote_functions -def desc_nulls_last(col: "ColumnOrName") -> Column: +def median(col: "ColumnOrName") -> Column: """ - Sort Function: Returns a sort expression based on the descending order of the given - column name, and null values appear after non-null values. - - .. versionadded:: 2.4.0 + Returns the median of the values in a group. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.4.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to sort by in the descending order. + target column to compute on. + A column that evaluates to a numeric, interval, or time. Returns ------- :class:`~pyspark.sql.Column` - the column specifying the order. + the median of the values in a group. + + Notes + ----- + Supports Spark Connect. + + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.approx_percentile` + :meth:`pyspark.sql.functions.percentile_approx` See Also -------- - :meth:`pyspark.sql.functions.desc` - :meth:`pyspark.sql.functions.desc_nulls_first` + :meth:`pyspark.sql.functions.approx_percentile` + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.percentile_approx` Examples -------- - Example 1: Sorting a DataFrame with null values in descending order - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_last(df.name)).show() - +---+-----+ - |age| name| - +---+-----+ - | 1| Bob| - | 2|Alice| - | 0| NULL| - +---+-----+ + >>> df = spark.createDataFrame([ + ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), + ... ("Java", 2012, 22000), ("dotNET", 2012, 10000), + ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], + ... schema=("course", "year", "earnings")) + >>> df.groupby("course").agg(sf.median("earnings")).show() + +------+----------------+ + |course|median(earnings)| + +------+----------------+ + | Java| 22000.0| + |dotNET| 10000.0| + +------+----------------+ + """ + return _invoke_function_over_columns("median", col) - Example 2: Sorting a DataFrame with multiple columns, null values in descending order - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) - >>> df.sort(sf.desc_nulls_last(df.name), sf.desc_nulls_last(df.grade)).show() - +---+-----+-----+ - |age| name|grade| - +---+-----+-----+ - | 1| Bob| NULL| - | 2|Alice| Y| - | 0| NULL| Z| - +---+-----+-----+ - - Example 3: Sorting a DataFrame with null values in descending order using column name string - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) - >>> df.sort(sf.desc_nulls_last("name")).show() - +---+-----+ - |age| name| - +---+-----+ - | 1| Bob| - | 2|Alice| - | 0| NULL| - +---+-----+ +@_try_remote_functions +def sumDistinct(col: "ColumnOrName") -> Column: """ - return ( - col.desc_nulls_last() - if isinstance(col, Column) - else _invoke_function("desc_nulls_last", col) - ) + Aggregate function: returns the sum of distinct values in the expression. + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. -# ---------------------- Mathematical Functions ---------------------- + .. deprecated:: 3.2.0 + Use :func:`sum_distinct` instead. + """ + warnings.warn("Deprecated in 3.2, use sum_distinct instead.", FutureWarning) + return sum_distinct(col) @_try_remote_functions -def sqrt(col: "ColumnOrName") -> Column: +def sum_distinct(col: "ColumnOrName") -> Column: """ - Computes the square root of the specified float value. + Aggregate function: returns the sum of distinct values in the expression. - .. versionadded:: 1.3.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -1832,389 +1919,381 @@ def sqrt(col: "ColumnOrName") -> Column: ---------- col : :class:`~pyspark.sql.Column` or column name target column to compute on. - A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - column for computed results. - Returns a column that evaluates to a double. + the column for computed results. Examples -------- + Example 1: Using sum_distinct function on a column with all distinct values + >>> from pyspark.sql import functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (-1), (0), (1), (4), (NULL) AS TAB(value)" - ... ).select("*", sf.sqrt("value")).show() - +-----+-----------+ - |value|SQRT(value)| - +-----+-----------+ - | -1| NaN| - | 0| 0.0| - | 1| 1.0| - | 4| 2.0| - | NULL| NULL| - +-----+-----------+ + >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,)], ["numbers"]) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | 10| + +---------------------+ + + Example 2: Using sum_distinct function on a column with no distinct values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,), (1,), (1,), (1,)], ["numbers"]) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | 1| + +---------------------+ + + Example 3: Using sum_distinct function on a column with null and duplicate values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(None,), (1,), (1,), (2,)], ["numbers"]) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | 3| + +---------------------+ + + Example 4: Using sum_distinct function on a column with all None values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, IntegerType + >>> schema = StructType([StructField("numbers", IntegerType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.sum_distinct('numbers')).show() + +---------------------+ + |sum(DISTINCT numbers)| + +---------------------+ + | NULL| + +---------------------+ """ - return _invoke_function_over_columns("sqrt", col) + return _invoke_function_over_columns("sum_distinct", col) @_try_remote_functions -def try_add(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None) -> Column: """ - Returns the sum of `left`and `right` and the result is null on overflow. - The acceptable input types are the same with the `+` operator. + Aggregate function: returns the concatenation of non-null input values, + separated by the delimiter. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. - right : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a string or binary. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. Examples -------- - Example 1: Integer plus Integer. - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(1982, 15), (1990, 2)], ["birth", "age"] - ... ).select("*", sf.try_add("birth", "age")).show() - +-----+---+-------------------+ - |birth|age|try_add(birth, age)| - +-----+---+-------------------+ - | 1982| 15| 1997| - | 1990| 2| 1992| - +-----+---+-------------------+ - - Example 2: Date plus Integer. + Example 1: Using listagg function - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-09-30')) AS TAB(date)" - ... ).select("*", sf.try_add("date", sf.lit(1))).show() - +----------+----------------+ - | date|try_add(date, 1)| - +----------+----------------+ - |2015-09-30| 2015-10-01| - +----------+----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.listagg('strings')).show() + +----------------------+ + |listagg(strings, NULL)| + +----------------------+ + | abc| + +----------------------+ - Example 3: Date plus Interval. + Example 2: Using listagg function with a delimiter - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" - ... ).select("*", sf.try_add("date", "itvl")).show() - +----------+-----------------+-------------------+ - | date| itvl|try_add(date, itvl)| - +----------+-----------------+-------------------+ - |2015-09-30|INTERVAL '1' YEAR| 2016-09-30| - +----------+-----------------+-------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.listagg('strings', ', ')).show() + +--------------------+ + |listagg(strings, , )| + +--------------------+ + | a, b, c| + +--------------------+ - Example 4: Interval plus Interval. + Example 3: Using listagg function with a binary column and delimiter - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" - ... ).select("*", sf.try_add("itvl1", "itvl2")).show() - +-----------------+-----------------+---------------------+ - | itvl1| itvl2|try_add(itvl1, itvl2)| - +-----------------+-----------------+---------------------+ - |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '3' YEAR| - +-----------------+-----------------+---------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) + >>> df.select(sf.listagg('bytes', b'\x42')).show() + +---------------------+ + |listagg(bytes, X'42')| + +---------------------+ + | [01 42 02 42 03]| + +---------------------+ - Example 5: Overflow results in NULL when ANSI mode is on + Example 4: Using listagg function on a column with all None values - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_add(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-------------------------------------------------+ - |try_add(9223372036854775807, 9223372036854775807)| - +-------------------------------------------------+ - | NULL| - +-------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.listagg('strings')).show() + +----------------------+ + |listagg(strings, NULL)| + +----------------------+ + | NULL| + +----------------------+ """ - return _invoke_function_over_columns("try_add", left, right) + if delimiter is None: + return _invoke_function_over_columns("listagg", col) + else: + return _invoke_function_over_columns("listagg", col, lit(delimiter)) @_try_remote_functions -def try_divide(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def listagg_distinct( + col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None +) -> Column: """ - Returns `dividend`/`divisor`. It always performs floating point division. Its result is - always null if `divisor` is 0. + Aggregate function: returns the concatenation of distinct non-null input values, + separated by the delimiter. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - dividend. - A column that evaluates to a numeric or interval. - right : :class:`~pyspark.sql.Column` or column name - divisor. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. Examples -------- - Example 1: Integer divided by Integer. + Example 1: Using listagg_distinct function - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(6000, 15), (1990, 2), (1234, 0)], ["a", "b"] - ... ).select("*", sf.try_divide("a", "b")).show() - +----+---+----------------+ - | a| b|try_divide(a, b)| - +----+---+----------------+ - |6000| 15| 400.0| - |1990| 2| 995.0| - |1234| 0| NULL| - +----+---+----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.listagg_distinct('strings')).show() + +-------------------------------+ + |listagg(DISTINCT strings, NULL)| + +-------------------------------+ + | abc| + +-------------------------------+ - Example 2: Interval divided by Integer. + Example 2: Using listagg_distinct function with a delimiter - >>> import pyspark.sql.functions as sf - >>> df = spark.range(4).select(sf.make_interval(sf.lit(1)).alias("itvl"), "id") - >>> df.select("*", sf.try_divide("itvl", "id")).show() - +-------+---+--------------------+ - | itvl| id|try_divide(itvl, id)| - +-------+---+--------------------+ - |1 years| 0| NULL| - |1 years| 1| 1 years| - |1 years| 2| 6 months| - |1 years| 3| 4 months| - +-------+---+--------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.listagg_distinct('strings', ', ')).show() + +-----------------------------+ + |listagg(DISTINCT strings, , )| + +-----------------------------+ + | a, b, c| + +-----------------------------+ - Example 3: Exception during division, resulting in NULL when ANSI mode is on + Example 3: Using listagg_distinct function with a binary column and delimiter - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_divide("id", sf.lit(0))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-----------------+ - |try_divide(id, 0)| - +-----------------+ - | NULL| - +-----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], + ... ['bytes']) + >>> df.select(sf.listagg_distinct('bytes', b'\x42')).show() + +------------------------------+ + |listagg(DISTINCT bytes, X'42')| + +------------------------------+ + | [01 42 02 42 03]| + +------------------------------+ + + Example 4: Using listagg_distinct function on a column with all None values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.listagg_distinct('strings')).show() + +-------------------------------+ + |listagg(DISTINCT strings, NULL)| + +-------------------------------+ + | NULL| + +-------------------------------+ """ - return _invoke_function_over_columns("try_divide", left, right) + if delimiter is None: + return _invoke_function_over_columns("listagg_distinct", col) + else: + return _invoke_function_over_columns("listagg_distinct", col, lit(delimiter)) @_try_remote_functions -def try_mod(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def string_agg( + col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None +) -> Column: """ - Returns the remainder after `dividend`/`divisor`. Its result is - always null if `divisor` is 0. + Aggregate function: returns the concatenation of non-null input values, + separated by the delimiter. + + An alias of :func:`listagg`. .. versionadded:: 4.0.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - dividend. - A column that evaluates to a numeric. - right : :class:`~pyspark.sql.Column` or column name - divisor. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a string or binary. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. + A column that evaluates to a string, binary, or null. Must be a constant. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. Examples -------- - Example 1: Integer divided by Integer. + Example 1: Using string_agg function - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(6000, 15), (3, 2), (1234, 0)], ["a", "b"] - ... ).select("*", sf.try_mod("a", "b")).show() - +----+---+-------------+ - | a| b|try_mod(a, b)| - +----+---+-------------+ - |6000| 15| 0| - | 3| 2| 1| - |1234| 0| NULL| - +----+---+-------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.string_agg('strings')).show() + +-------------------------+ + |string_agg(strings, NULL)| + +-------------------------+ + | abc| + +-------------------------+ - Example 2: Exception during division, resulting in NULL when ANSI mode is on + Example 2: Using string_agg function with a delimiter - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_mod("id", sf.lit(0))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +--------------+ - |try_mod(id, 0)| - +--------------+ - | NULL| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) + >>> df.select(sf.string_agg('strings', ', ')).show() + +-----------------------+ + |string_agg(strings, , )| + +-----------------------+ + | a, b, c| + +-----------------------+ + + Example 3: Using string_agg function with a binary column and delimiter + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) + >>> df.select(sf.string_agg('bytes', b'\x42')).show() + +------------------------+ + |string_agg(bytes, X'42')| + +------------------------+ + | [01 42 02 42 03]| + +------------------------+ + + Example 4: Using string_agg function on a column with all None values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.string_agg('strings')).show() + +-------------------------+ + |string_agg(strings, NULL)| + +-------------------------+ + | NULL| + +-------------------------+ """ - return _invoke_function_over_columns("try_mod", left, right) + if delimiter is None: + return _invoke_function_over_columns("string_agg", col) + else: + return _invoke_function_over_columns("string_agg", col, lit(delimiter)) @_try_remote_functions -def try_multiply(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def string_agg_distinct( + col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None +) -> Column: """ - Returns `left`*`right` and the result is null on overflow. The acceptable input types are the - same with the `*` operator. + Aggregate function: returns the concatenation of distinct non-null input values, + separated by the delimiter. - .. versionadded:: 3.5.0 + An alias of :func:`listagg_distinct`. + + .. versionadded:: 4.0.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - multiplicand. - A column that evaluates to a numeric or interval. - right : :class:`~pyspark.sql.Column` or column name - multiplier. - A column that evaluates to a numeric or interval. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional + the delimiter to separate the values. The default value is None. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. Examples -------- - Example 1: Integer multiplied by Integer. + Example 1: Using string_agg_distinct function - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(6000, 15), (1990, 2)], ["a", "b"] - ... ).select("*", sf.try_multiply("a", "b")).show() - +----+---+------------------+ - | a| b|try_multiply(a, b)| - +----+---+------------------+ - |6000| 15| 90000| - |1990| 2| 3980| - +----+---+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.string_agg_distinct('strings')).show() + +----------------------------------+ + |string_agg(DISTINCT strings, NULL)| + +----------------------------------+ + | abc| + +----------------------------------+ - Example 2: Interval multiplied by Integer. + Example 2: Using string_agg_distinct function with a delimiter - >>> import pyspark.sql.functions as sf - >>> df = spark.range(6).select(sf.make_interval(sf.col("id"), sf.lit(3)).alias("itvl"), "id") - >>> df.select("*", sf.try_multiply("itvl", "id")).show() - +----------------+---+----------------------+ - | itvl| id|try_multiply(itvl, id)| - +----------------+---+----------------------+ - | 3 months| 0| 0 seconds| - |1 years 3 months| 1| 1 years 3 months| - |2 years 3 months| 2| 4 years 6 months| - |3 years 3 months| 3| 9 years 9 months| - |4 years 3 months| 4| 17 years| - |5 years 3 months| 5| 26 years 3 months| - +----------------+---+----------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) + >>> df.select(sf.string_agg_distinct('strings', ', ')).show() + +--------------------------------+ + |string_agg(DISTINCT strings, , )| + +--------------------------------+ + | a, b, c| + +--------------------------------+ - Example 3: Overflow results in NULL when ANSI mode is on + Example 3: Using string_agg_distinct function with a binary column and delimiter - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_multiply(sf.lit(sys.maxsize), sf.lit(sys.maxsize))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +------------------------------------------------------+ - |try_multiply(9223372036854775807, 9223372036854775807)| - +------------------------------------------------------+ - | NULL| - +------------------------------------------------------+ - """ - return _invoke_function_over_columns("try_multiply", left, right) - - -@_try_remote_functions -def try_subtract(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """ - Returns `left`-`right` and the result is null on overflow. The acceptable input types are the - same with the `-` operator. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - left : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. - right : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric, interval, date, timestamp, or time. - - Examples - -------- - Example 1: Integer minus Integer. - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(1982, 15), (1990, 2)], ["birth", "age"] - ... ).select("*", sf.try_subtract("birth", "age")).show() - +-----+---+------------------------+ - |birth|age|try_subtract(birth, age)| - +-----+---+------------------------+ - | 1982| 15| 1967| - | 1990| 2| 1988| - +-----+---+------------------------+ - - Example 2: Date minus Integer. - - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-10-01')) AS TAB(date)" - ... ).select("*", sf.try_subtract("date", sf.lit(1))).show() - +----------+---------------------+ - | date|try_subtract(date, 1)| - +----------+---------------------+ - |2015-10-01| 2015-09-30| - +----------+---------------------+ - - Example 3: Date minus Interval. - - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (DATE('2015-09-30'), INTERVAL 1 YEAR) AS TAB(date, itvl)" - ... ).select("*", sf.try_subtract("date", "itvl")).show() - +----------+-----------------+------------------------+ - | date| itvl|try_subtract(date, itvl)| - +----------+-----------------+------------------------+ - |2015-09-30|INTERVAL '1' YEAR| 2014-09-30| - +----------+-----------------+------------------------+ - - Example 4: Interval minus Interval. - - >>> import pyspark.sql.functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (INTERVAL 1 YEAR, INTERVAL 2 YEAR) AS TAB(itvl1, itvl2)" - ... ).select("*", sf.try_subtract("itvl1", "itvl2")).show() - +-----------------+-----------------+--------------------------+ - | itvl1| itvl2|try_subtract(itvl1, itvl2)| - +-----------------+-----------------+--------------------------+ - |INTERVAL '1' YEAR|INTERVAL '2' YEAR| INTERVAL '-1' YEAR| - +-----------------+-----------------+--------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], + ... ['bytes']) + >>> df.select(sf.string_agg_distinct('bytes', b'\x42')).show() + +---------------------------------+ + |string_agg(DISTINCT bytes, X'42')| + +---------------------------------+ + | [01 42 02 42 03]| + +---------------------------------+ - Example 5: Overflow results in NULL when ANSI mode is on + Example 4: Using string_agg_distinct function on a column with all None values - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... spark.range(1).select(sf.try_subtract(sf.lit(-sys.maxsize), sf.lit(sys.maxsize))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-------------------------------------------------------+ - |try_subtract(-9223372036854775807, 9223372036854775807)| - +-------------------------------------------------------+ - | NULL| - +-------------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import StructType, StructField, StringType + >>> schema = StructType([StructField("strings", StringType(), True)]) + >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) + >>> df.select(sf.string_agg_distinct('strings')).show() + +----------------------------------+ + |string_agg(DISTINCT strings, NULL)| + +----------------------------------+ + | NULL| + +----------------------------------+ """ - return _invoke_function_over_columns("try_subtract", left, right) + if delimiter is None: + return _invoke_function_over_columns("string_agg_distinct", col) + else: + return _invoke_function_over_columns("string_agg_distinct", col, lit(delimiter)) @_try_remote_functions -def abs(col: "ColumnOrName") -> Column: +def product(col: "ColumnOrName") -> Column: """ - Mathematical Function: Computes the absolute value of the given column or expression. + Aggregate function: returns the product of the values in a group. - .. versionadded:: 1.3.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -2222,59 +2301,27 @@ def abs(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The target column or expression to compute the absolute value on. - A column that evaluates to a numeric or interval. + column containing values to be multiplied together Returns ------- - :class:`~pyspark.sql.Column` - A new column object representing the absolute value of the input. - Returns a column of the same type as the input. + :class:`~pyspark.sql.Column` or column name + the column for computed results. Examples -------- - Example 1: Compute the absolute value of a long column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(-1,), (-2,), (-3,), (None,)], ["value"]) - >>> df.select("*", sf.abs(df.value)).show() - +-----+----------+ - |value|abs(value)| - +-----+----------+ - | -1| 1| - | -2| 2| - | -3| 3| - | NULL| NULL| - +-----+----------+ - - Example 2: Compute the absolute value of a double column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(-1.5,), (-2.5,), (None,), (float("nan"),)], ["value"]) - >>> df.select("*", sf.abs(df.value)).show() - +-----+----------+ - |value|abs(value)| - +-----+----------+ - | -1.5| 1.5| - | -2.5| 2.5| - | NULL| NULL| - | NaN| NaN| - +-----+----------+ - - Example 3: Compute the absolute value of an expression - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1), (2, -2), (3, 3)], ["id", "value"]) - >>> df.select("*", sf.abs(df.id - df.value)).show() - +---+-----+-----------------+ - | id|value|abs((id - value))| - +---+-----+-----------------+ - | 1| 1| 0| - | 2| -2| 4| - | 3| 3| 0| - +---+-----+-----------------+ + >>> df = spark.sql("SELECT id % 3 AS mod3, id AS value FROM RANGE(10)") + >>> df.groupBy('mod3').agg(sf.product('value')).orderBy('mod3').show() + +----+--------------+ + |mod3|product(value)| + +----+--------------+ + | 0| 0.0| + | 1| 28.0| + | 2| 80.0| + +----+--------------+ """ - return _invoke_function_over_columns("abs", col) + return _invoke_function_over_columns("product", col) @_try_remote_functions @@ -3873,12 +3920,28 @@ def toRadians(col: "ColumnOrName") -> Column: @_try_remote_functions -def degrees(col: "ColumnOrName") -> Column: +def bitwiseNOT(col: "ColumnOrName") -> Column: """ - Converts an angle measured in radians to an approximately equivalent angle - measured in degrees. + Computes bitwise not. - .. versionadded:: 2.1.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 3.2.0 + Use :func:`bitwise_not` instead. + """ + warnings.warn("Deprecated in 3.2, use bitwise_not instead.", FutureWarning) + return bitwise_not(col) + + +@_try_remote_functions +def bitwise_not(col: "ColumnOrName") -> Column: + """ + Computes bitwise not. + + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -3886,456 +3949,505 @@ def degrees(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in radians. - A column that evaluates to a double. + target column to compute on. Returns ------- :class:`~pyspark.sql.Column` - angle in degrees, as if computed by `java.lang.Math.toDegrees()` - Returns a column that evaluates to a double. - - See Also - -------- - :meth:`pyspark.sql.functions.radians` + the column for computed results. Examples -------- >>> from pyspark.sql import functions as sf >>> spark.sql( - ... "SELECT * FROM VALUES (0.0), (PI()), (PI() / 2), (PI() / 4) AS TAB(value)" - ... ).select("*", sf.degrees("value")).show() - +------------------+--------------+ - | value|DEGREES(value)| - +------------------+--------------+ - | 0.0| 0.0| - | 3.141592653589...| 180.0| - |1.5707963267948...| 90.0| - |0.7853981633974...| 45.0| - +------------------+--------------+ + ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" + ... ).select("*", sf.bitwise_not("value")).show() + +-----+------+ + |value|~value| + +-----+------+ + | 0| -1| + | 1| -2| + | 2| -3| + | 3| -4| + | NULL| NULL| + +-----+------+ """ - return _invoke_function_over_columns("degrees", col) + return _invoke_function_over_columns("bitwise_not", col) @_try_remote_functions -def radians(col: "ColumnOrName") -> Column: +def bit_count(col: "ColumnOrName") -> Column: """ - Converts an angle measured in degrees to an approximately equivalent angle - measured in radians. - - .. versionadded:: 2.1.0 + Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer, + or NULL if the argument is NULL. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - angle in degrees. - A column that evaluates to a double. + target column to compute on. + A column that evaluates to an integral or boolean. Returns ------- :class:`~pyspark.sql.Column` - angle in radians, as if computed by `java.lang.Math.toRadians()` - Returns a column that evaluates to a double. + the number of bits that are set in the argument expr as an unsigned 64-bit integer, + or NULL if the argument is NULL. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.degrees` + :meth:`pyspark.sql.functions.bit_get` Examples -------- >>> from pyspark.sql import functions as sf >>> spark.sql( - ... "SELECT * FROM VALUES (180), (90), (45), (0) AS TAB(value)" - ... ).select("*", sf.radians("value")).show() - +-----+------------------+ - |value| RADIANS(value)| - +-----+------------------+ - | 180| 3.141592653589...| - | 90|1.5707963267948...| - | 45|0.7853981633974...| - | 0| 0.0| - +-----+------------------+ + ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" + ... ).select("*", sf.bit_count("value")).show() + +-----+----------------+ + |value|bit_count(value)| + +-----+----------------+ + | 0| 0| + | 1| 1| + | 2| 1| + | 3| 2| + | NULL| NULL| + +-----+----------------+ """ - return _invoke_function_over_columns("radians", col) + return _invoke_function_over_columns("bit_count", col) @_try_remote_functions -def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: +def bit_get(col: "ColumnOrName", pos: "ColumnOrName") -> Column: """ - Compute the angle in radians between the positive x-axis of a plane - and the point given by the coordinates - - .. versionadded:: 1.4.0 + Returns the value of the bit (0 or 1) at the specified position. + The positions are numbered from right to left, starting at zero. + The position argument cannot be negative. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column`, column name or float - coordinate on y-axis. - A column that evaluates to a double. - col2 : :class:`~pyspark.sql.Column`, column name or float - coordinate on x-axis. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. + pos : :class:`~pyspark.sql.Column` or column name + The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - the `theta` component of the point - (`r`, `theta`) - in polar coordinates that corresponds to the point - (`x`, `y`) in Cartesian coordinates, - as if computed by `java.lang.Math.atan2()` - Returns a column that evaluates to a double. + the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. See Also -------- - :meth:`pyspark.sql.functions.atan` - :meth:`pyspark.sql.functions.hypot` + :meth:`pyspark.sql.functions.bit_count` + :meth:`pyspark.sql.functions.getbit` Examples -------- + Example 1: Get the bit with a literal position + >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.atan2(sf.lit(1), sf.lit(2))).show() - +------------------+ - | ATAN2(1, 2)| - +------------------+ - |0.4636476090008...| - +------------------+ + >>> df = spark.createDataFrame([[1],[2],[3],[None]], ["value"]) + >>> df.select("*", sf.bit_get("value", sf.lit(1))).show() + +-----+-----------------+ + |value|bit_get(value, 1)| + +-----+-----------------+ + | 1| 0| + | 2| 1| + | 3| 1| + | NULL| NULL| + +-----+-----------------+ + + Example 2: Get the bit with a column position + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) + >>> df.select("*", sf.bit_get(df.value, "pos")).show() + +-----+----+-------------------+ + |value| pos|bit_get(value, pos)| + +-----+----+-------------------+ + | 1| 2| 0| + | 2| 1| 1| + | 3|NULL| NULL| + | NULL| 1| NULL| + +-----+----+-------------------+ """ - return _invoke_binary_math_function("atan2", col1, col2) + return _invoke_function_over_columns("bit_get", col, pos) @_try_remote_functions -def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: +def getbit(col: "ColumnOrName", pos: "ColumnOrName") -> Column: """ - Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow. - - .. versionadded:: 1.4.0 + Returns the value of the bit (0 or 1) at the specified position. + The positions are numbered from right to left, starting at zero. + The position argument cannot be negative. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column`, column name or float - a leg. - A column that evaluates to a double. - col2 : :class:`~pyspark.sql.Column`, column name or float - b leg. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. + pos : :class:`~pyspark.sql.Column` or column name + The positions are numbered from right to left, starting at zero. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - length of the hypotenuse. - Returns a column that evaluates to a double. + the value of the bit (0 or 1) at the specified position. + Returns a column that evaluates to a byte. + + See Also + -------- + :meth:`pyspark.sql.functions.bit_get` + :meth:`pyspark.sql.functions.bit_count` Examples -------- + Example 1: Get the bit with a literal position + + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[1], [2], [3], [None]], ["value"] + ... ).select("*", sf.getbit("value", sf.lit(1))).show() + +-----+----------------+ + |value|getbit(value, 1)| + +-----+----------------+ + | 1| 0| + | 2| 1| + | 3| 1| + | NULL| NULL| + +-----+----------------+ + + Example 2: Get the bit with a column position + >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.hypot(sf.lit(1), sf.lit(2))).show() - +----------------+ - | HYPOT(1, 2)| - +----------------+ - |2.23606797749...| - +----------------+ + >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) + >>> df.select("*", sf.getbit(df.value, "pos")).show() + +-----+----+------------------+ + |value| pos|getbit(value, pos)| + +-----+----+------------------+ + | 1| 2| 0| + | 2| 1| 1| + | 3|NULL| NULL| + | NULL| 1| NULL| + +-----+----+------------------+ """ - return _invoke_binary_math_function("hypot", col1, col2) + return _invoke_function_over_columns("getbit", col, pos) @_try_remote_functions -def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: +def asc_nulls_first(col: "ColumnOrName") -> Column: """ - Returns the value of the first argument raised to the power of the second argument. + Sort Function: Returns a sort expression based on the ascending order of the given + column name, and null values return before non-null values. - .. versionadded:: 1.4.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col1 : :class:`~pyspark.sql.Column`, column name or float - the base number. - A column that evaluates to a double. - col2 : :class:`~pyspark.sql.Column`, column name or float - the exponent number. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to sort by in the ascending order. Returns ------- :class:`~pyspark.sql.Column` - the base rased to the power the argument. - Returns a column that evaluates to a double. + the column specifying the order. + + See Also + -------- + :meth:`pyspark.sql.functions.asc` + :meth:`pyspark.sql.functions.asc_nulls_last` Examples -------- + Example 1: Sorting a DataFrame with null values in ascending order + >>> from pyspark.sql import functions as sf - >>> spark.range(5).select("*", sf.pow("id", 2)).show() - +---+------------+ - | id|POWER(id, 2)| - +---+------------+ - | 0| 0.0| - | 1| 1.0| - | 2| 4.0| - | 3| 9.0| - | 4| 16.0| - +---+------------+ - """ - return _invoke_binary_math_function("pow", col1, col2) + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_first(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 2|Alice| + | 1| Bob| + +---+-----+ + Example 2: Sorting a DataFrame with multiple columns, null values in ascending order -power = pow + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.asc_nulls_first(df.name), sf.asc_nulls_first(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 0| NULL| Z| + | 2|Alice| Y| + | 1| Bob| NULL| + +---+-----+-----+ + + Example 3: Sorting a DataFrame with null values in ascending order using column name string + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_first("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 2|Alice| + | 1| Bob| + +---+-----+ + """ + return ( + col.asc_nulls_first() + if isinstance(col, Column) + else _invoke_function("asc_nulls_first", col) + ) @_try_remote_functions -def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", float]) -> Column: +def asc_nulls_last(col: "ColumnOrName") -> Column: """ - Returns the positive value of dividend mod divisor. + Sort Function: Returns a sort expression based on the ascending order of the given + column name, and null values appear after non-null values. - .. versionadded:: 3.4.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - dividend : :class:`~pyspark.sql.Column`, column name or float - the column that contains dividend, or the specified dividend value. - A column that evaluates to a numeric. - divisor : :class:`~pyspark.sql.Column`, column name or float - the column that contains divisor, or the specified divisor value. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or column name + target column to sort by in the ascending order. Returns ------- :class:`~pyspark.sql.Column` - positive value of dividend mod divisor. - Returns a column of the same type as the input. + the column specifying the order. - Notes - ----- - Supports Spark Connect. + See Also + -------- + :meth:`pyspark.sql.functions.asc` + :meth:`pyspark.sql.functions.asc_nulls_first` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1.0, float('nan')), (float('nan'), 2.0), (10.0, 3.0), - ... (float('nan'), float('nan')), (-3.0, 4.0), (-10.0, 3.0), - ... (-5.0, -6.0), (7.0, -8.0), (1.0, 2.0)], - ... ("a", "b")) - >>> df.select("*", sf.pmod("a", "b")).show() - +-----+----+----------+ - | a| b|pmod(a, b)| - +-----+----+----------+ - | 1.0| NaN| NaN| - | NaN| 2.0| NaN| - | 10.0| 3.0| 1.0| - | NaN| NaN| NaN| - | -3.0| 4.0| 1.0| - |-10.0| 3.0| 2.0| - | -5.0|-6.0| -5.0| - | 7.0|-8.0| 7.0| - | 1.0| 2.0| 1.0| - +-----+----+----------+ - """ - return _invoke_binary_math_function("pmod", dividend, divisor) - + Example 1: Sorting a DataFrame with null values in ascending order -@_try_remote_functions -def width_bucket( - v: "ColumnOrName", - min: "ColumnOrName", - max: "ColumnOrName", - numBucket: Union["ColumnOrName", int], -) -> Column: - """ - Returns the bucket number into which the value of this expression would fall - after being evaluated. Note that input arguments must follow conditions listed below; - otherwise, the method will return null. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_last(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 2|Alice| + | 1| Bob| + | 0| NULL| + +---+-----+ - .. versionadded:: 3.5.0 + Example 2: Sorting a DataFrame with multiple columns, null values in ascending order - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or column name - value to compute a bucket number in the histogram. - A column that evaluates to a double or interval. - min : :class:`~pyspark.sql.Column` or column name - minimum value of the histogram. - A column that evaluates to a double or interval. - max : :class:`~pyspark.sql.Column` or column name - maximum value of the histogram. - A column that evaluates to a double or interval. - numBucket : :class:`~pyspark.sql.Column`, column name or int - the number of buckets. - A column that evaluates to a long. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.asc_nulls_last(df.name), sf.asc_nulls_last(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 2|Alice| Y| + | 1| Bob| NULL| + | 0| NULL| Z| + +---+-----+-----+ - Returns - ------- - :class:`~pyspark.sql.Column` - the bucket number into which the value would fall after being evaluated - Returns a column that evaluates to a long. + Example 3: Sorting a DataFrame with null values in ascending order using column name string - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (5.3, 0.2, 10.6, 5), - ... (-2.1, 1.3, 3.4, 3), - ... (8.1, 0.0, 5.7, 4), - ... (-0.9, 5.2, 0.5, 2)], - ... ['v', 'min', 'max', 'n']) - >>> df.select("*", sf.width_bucket('v', 'min', 'max', 'n')).show() - +----+---+----+---+----------------------------+ - | v|min| max| n|width_bucket(v, min, max, n)| - +----+---+----+---+----------------------------+ - | 5.3|0.2|10.6| 5| 3| - |-2.1|1.3| 3.4| 3| 0| - | 8.1|0.0| 5.7| 4| 5| - |-0.9|5.2| 0.5| 2| 3| - +----+---+----+---+----------------------------+ + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.asc_nulls_last("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 2|Alice| + | 1| Bob| + | 0| NULL| + +---+-----+ """ - numBucket = _enum_to_value(numBucket) - numBucket = lit(numBucket) if isinstance(numBucket, int) else numBucket - return _invoke_function_over_columns("width_bucket", v, min, max, numBucket) + return ( + col.asc_nulls_last() if isinstance(col, Column) else _invoke_function("asc_nulls_last", col) + ) @_try_remote_functions -def rand(seed: Optional[int] = None) -> Column: - """Generates a random column with independent and identically distributed (i.i.d.) samples - uniformly distributed in [0.0, 1.0). +def desc_nulls_first(col: "ColumnOrName") -> Column: + """ + Sort Function: Returns a sort expression based on the descending order of the given + column name, and null values appear before non-null values. - .. versionadded:: 1.4.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Notes - ----- - The function is non-deterministic in general case. - Parameters ---------- - seed : int, optional - Seed value for the random generator. - A column that evaluates to an integer or long. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + target column to sort by in the descending order. Returns ------- :class:`~pyspark.sql.Column` - A column of random values. - Returns a column that evaluates to a double. + the column specifying the order. See Also -------- - :meth:`pyspark.sql.functions.randn` - :meth:`pyspark.sql.functions.randstr` - :meth:`pyspark.sql.functions.uniform` + :meth:`pyspark.sql.functions.desc` + :meth:`pyspark.sql.functions.desc_nulls_last` Examples -------- - Example 1: Generate a random column without a seed + Example 1: Sorting a DataFrame with null values in descending order >>> from pyspark.sql import functions as sf - >>> spark.range(0, 2, 1, 1).select("*", sf.rand()).show() # doctest: +SKIP - +---+-------------------------+ - | id|rand(-158884697681280011)| - +---+-------------------------+ - | 0| 0.9253464547887...| - | 1| 0.6533254118758...| - +---+-------------------------+ + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_first(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 1| Bob| + | 2|Alice| + +---+-----+ - Example 2: Generate a random column with a specific seed + Example 2: Sorting a DataFrame with multiple columns, null values in descending order - >>> spark.range(0, 2, 1, 1).select("*", sf.rand(seed=42)).show() - +---+------------------+ - | id| rand(42)| - +---+------------------+ - | 0| 0.619189370225...| - | 1|0.5096018842446...| - +---+------------------+ - """ - if seed is not None: - return _invoke_function("rand", _enum_to_value(seed)) - else: - return _invoke_function("rand") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(1, "Bob", None), (0, None, "Z"), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.desc_nulls_first(df.name), sf.desc_nulls_first(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 0| NULL| Z| + | 1| Bob| NULL| + | 2|Alice| Y| + +---+-----+-----+ + Example 3: Sorting a DataFrame with null values in descending order using column name string -random = rand + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "Bob"), (0, None), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_first("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 0| NULL| + | 1| Bob| + | 2|Alice| + +---+-----+ + """ + return ( + col.desc_nulls_first() + if isinstance(col, Column) + else _invoke_function("desc_nulls_first", col) + ) @_try_remote_functions -def randn(seed: Optional[int] = None) -> Column: - """Generates a random column with independent and identically distributed (i.i.d.) samples - from the standard normal distribution. +def desc_nulls_last(col: "ColumnOrName") -> Column: + """ + Sort Function: Returns a sort expression based on the descending order of the given + column name, and null values appear after non-null values. - .. versionadded:: 1.4.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Notes - ----- - The function is non-deterministic in general case. - Parameters ---------- - seed : int (default: None) - Seed value for the random generator. - A column that evaluates to an integer or long. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + target column to sort by in the descending order. Returns ------- :class:`~pyspark.sql.Column` - A column of random values. - Returns a column that evaluates to a double. + the column specifying the order. See Also -------- - :meth:`pyspark.sql.functions.rand` - :meth:`pyspark.sql.functions.randstr` - :meth:`pyspark.sql.functions.uniform` + :meth:`pyspark.sql.functions.desc` + :meth:`pyspark.sql.functions.desc_nulls_first` Examples -------- - Example 1: Generate a random column without a seed + Example 1: Sorting a DataFrame with null values in descending order >>> from pyspark.sql import functions as sf - >>> spark.range(0, 2, 1, 1).select("*", sf.randn()).show() # doctest: +SKIP - +---+--------------------------+ - | id|randn(3968742514375399317)| - +---+--------------------------+ - | 0| -0.47968645355788...| - | 1| -0.4950952457305...| - +---+--------------------------+ + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_last(df.name)).show() + +---+-----+ + |age| name| + +---+-----+ + | 1| Bob| + | 2|Alice| + | 0| NULL| + +---+-----+ - Example 2: Generate a random column with a specific seed + Example 2: Sorting a DataFrame with multiple columns, null values in descending order - >>> spark.range(0, 2, 1, 1).select("*", sf.randn(seed=42)).show() - +---+------------------+ - | id| randn(42)| - +---+------------------+ - | 0| 2.384479054241...| - | 1|0.1920934041293...| - +---+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(0, None, "Z"), (1, "Bob", None), (2, "Alice", "Y")], ["age", "name", "grade"]) + >>> df.sort(sf.desc_nulls_last(df.name), sf.desc_nulls_last(df.grade)).show() + +---+-----+-----+ + |age| name|grade| + +---+-----+-----+ + | 1| Bob| NULL| + | 2|Alice| Y| + | 0| NULL| Z| + +---+-----+-----+ + + Example 3: Sorting a DataFrame with null values in descending order using column name string + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(0, None), (1, "Bob"), (2, "Alice")], ["age", "name"]) + >>> df.sort(sf.desc_nulls_last("name")).show() + +---+-----+ + |age| name| + +---+-----+ + | 1| Bob| + | 2|Alice| + | 0| NULL| + +---+-----+ """ - if seed is not None: - return _invoke_function("randn", _enum_to_value(seed)) - else: - return _invoke_function("randn") + return ( + col.desc_nulls_last() + if isinstance(col, Column) + else _invoke_function("desc_nulls_last", col) + ) @_try_remote_functions -def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: +def stddev(col: "ColumnOrName") -> Column: """ - Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 - or at integral part when `scale` < 0. + Aggregate function: alias for stddev_samp. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -4343,119 +4455,85 @@ def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Co Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The target column or column name to compute the round on. + target column to compute on. A column that evaluates to a numeric. - scale : :class:`~pyspark.sql.Column` or int, optional - An optional parameter to control the rounding behavior. - A column that evaluates to an integer. Must be a constant. - .. versionchanged:: 4.0.0 - Support Column type. + See Also + -------- + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.skewness` + :meth:`pyspark.sql.functions.kurtosis` Returns ------- :class:`~pyspark.sql.Column` - A column for the rounded value. - Returns a column of the same type as the input. + standard deviation of given column. + Returns a column that evaluates to a double. Examples -------- - Example 1: Compute the rounded of a column value - - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.round(sf.lit(2.5))).show() - +-------------+ - |round(2.5, 0)| - +-------------+ - | 3.0| - +-------------+ - - Example 2: Compute the rounded of a column value with a specified scale - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.round(sf.lit(2.1267), sf.lit(2))).show() - +----------------+ - |round(2.1267, 2)| - +----------------+ - | 2.13| - +----------------+ + >>> spark.range(6).select(sf.stddev("id")).show() + +------------------+ + | stddev(id)| + +------------------+ + |1.8708286933869...| + +------------------+ """ - if scale is None: - return _invoke_function_over_columns("round", col) - else: - scale = _enum_to_value(scale) - scale = lit(scale) if isinstance(scale, int) else scale - return _invoke_function_over_columns("round", col, scale) + return _invoke_function_over_columns("stddev", col) @_try_remote_functions -def truncate(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: +def std(col: "ColumnOrName") -> Column: """ - Truncate the given value toward zero to `scale` decimal places when `scale` >= 0, - or to the left of the decimal point when `scale` < 0. `scale` defaults to 0. - - Unlike :func:`round`, the result is always rounded toward zero, and unlike :func:`floor` - negative values are not rounded toward negative infinity. + Aggregate function: alias for stddev_samp. - .. versionadded:: 4.4.0 + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The target column or column name to truncate. + target column to compute on. A column that evaluates to a numeric. - scale : :class:`~pyspark.sql.Column` or int, optional - An optional parameter to control the number of decimal places to keep. - A column that evaluates to an integer. Must be a constant. Defaults to 0. Returns ------- :class:`~pyspark.sql.Column` - A column for the truncated value, of the same type as the input, except that a decimal - input may return a decimal of different precision and scale. + standard deviation of given column. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.round` - :meth:`pyspark.sql.functions.trunc` - :meth:`pyspark.sql.functions.floor` - :meth:`pyspark.sql.functions.ceil` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.skewness` + :meth:`pyspark.sql.functions.kurtosis` Examples -------- - Example 1: Truncate toward zero to a given number of decimal places - - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.truncate(sf.lit(15.79), sf.lit(1)).alias("r")).collect() - [Row(r=15.7)] - - Example 2: Truncation rounds toward zero for negative values - - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.truncate(sf.lit(-2.99), sf.lit(0)).alias("r")).collect() - [Row(r=-2.0)] - - Example 3: The scale argument defaults to 0 when omitted - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.truncate(sf.lit(1234.5678)).alias("r")).collect() - [Row(r=1234.0)] + >>> spark.range(6).select(sf.std("id")).show() + +------------------+ + | std(id)| + +------------------+ + |1.8708286933869...| + +------------------+ """ - if scale is None: - return _invoke_function_over_columns("truncate", col) - else: - scale = _enum_to_value(scale) - scale = lit(scale) if isinstance(scale, int) else scale - return _invoke_function_over_columns("truncate", col, scale) + return _invoke_function_over_columns("std", col) @_try_remote_functions -def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: +def stddev_samp(col: "ColumnOrName") -> Column: """ - Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 - or at integral part when `scale` < 0. + Aggregate function: returns the unbiased sample standard deviation of + the expression in a group. - .. versionadded:: 2.0.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -4463,1456 +4541,1479 @@ def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> C Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The target column or column name to compute the round on. + target column to compute on. A column that evaluates to a numeric. - scale : :class:`~pyspark.sql.Column` or int, optional - An optional parameter to control the rounding behavior. - A column that evaluates to an integer. Must be a constant. - - .. versionchanged:: 4.0.0 - Support Column type. Returns ------- :class:`~pyspark.sql.Column` - A column for the rounded value. - Returns a column of the same type as the input. + standard deviation of given column. + Returns a column that evaluates to a double. - Examples + See Also -------- - Example 1: Compute the rounded of a column value - - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.bround(sf.lit(2.5))).show() - +--------------+ - |bround(2.5, 0)| - +--------------+ - | 2.0| - +--------------+ - - Example 2: Compute the rounded of a column value with a specified scale + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.var_samp` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.bround(sf.lit(2.1267), sf.lit(2))).show() - +-----------------+ - |bround(2.1267, 2)| - +-----------------+ - | 2.13| - +-----------------+ + >>> spark.range(6).select(sf.stddev_samp("id")).show() + +------------------+ + | stddev_samp(id)| + +------------------+ + |1.8708286933869...| + +------------------+ """ - if scale is None: - return _invoke_function_over_columns("bround", col) - else: - scale = _enum_to_value(scale) - scale = lit(scale) if isinstance(scale, int) else scale - return _invoke_function_over_columns("bround", col, scale) + return _invoke_function_over_columns("stddev_samp", col) @_try_remote_functions -def greatest(*cols: "ColumnOrName") -> Column: +def stddev_pop(col: "ColumnOrName") -> Column: """ - Returns the greatest value of the list of column names, skipping null values. - This function takes at least 2 parameters. It will return null if all parameters are null. + Aggregate function: returns population standard deviation of + the expression in a group. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - cols: :class:`~pyspark.sql.Column` or column name - columns to check for greatest value. - Each a column of any orderable type. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - greatest value. - Returns a column of the same type as the input. + standard deviation of given column. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.least` + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.var_pop` Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) - >>> df.select("*", sf.greatest(df.a, "b", df.c)).show() - +---+---+---+-----------------+ - | a| b| c|greatest(a, b, c)| - +---+---+---+-----------------+ - | 1| 4| 3| 4| - +---+---+---+-----------------+ + >>> spark.range(6).select(sf.stddev_pop("id")).show() + +-----------------+ + | stddev_pop(id)| + +-----------------+ + |1.707825127659...| + +-----------------+ """ - if len(cols) < 2: - raise PySparkValueError( - errorClass="WRONG_NUM_COLUMNS", - messageParameters={"func_name": "greatest", "num_cols": "2"}, - ) - return _invoke_function_over_seq_of_columns("greatest", cols) + return _invoke_function_over_columns("stddev_pop", col) @_try_remote_functions -def least(*cols: "ColumnOrName") -> Column: +def variance(col: "ColumnOrName") -> Column: """ - Returns the least value of the list of column names, skipping null values. - This function takes at least 2 parameters. It will return null if all parameters are null. + Aggregate function: alias for var_samp - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - column names or columns to be compared - Each a column of any orderable type. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - least value. - Returns a column of the same type as the input. + variance of given column. See Also -------- - :meth:`pyspark.sql.functions.greatest` + :meth:`pyspark.sql.functions.var_pop` + :meth:`pyspark.sql.functions.var_samp` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.skewness` + :meth:`pyspark.sql.functions.kurtosis` + :meth:`pyspark.sql.functions.std` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) - >>> df.select("*", sf.least(df.a, "b", df.c)).show() - +---+---+---+--------------+ - | a| b| c|least(a, b, c)| - +---+---+---+--------------+ - | 1| 4| 3| 1| - +---+---+---+--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.range(6) + >>> df.select(sf.variance(df.id)).show() + +------------+ + |variance(id)| + +------------+ + | 3.5| + +------------+ """ - if len(cols) < 2: - raise PySparkValueError( - errorClass="WRONG_NUM_COLUMNS", - messageParameters={"func_name": "least", "num_cols": "2"}, - ) - return _invoke_function_over_seq_of_columns("least", cols) - - -@overload -def log(arg1: "ColumnOrName") -> Column: ... - - -@overload -def log(arg1: float, arg2: "ColumnOrName") -> Column: ... + return _invoke_function_over_columns("variance", col) @_try_remote_functions -def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = None) -> Column: - """Returns the first argument-based logarithm of the second argument. - - If there is only one argument, then this takes the natural logarithm of the argument. +def var_samp(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns the unbiased sample variance of + the values in a group. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - arg1 : :class:`~pyspark.sql.Column`, str or float - base number or actual number (in this case base is `e`). - A column that evaluates to a double. - arg2 : :class:`~pyspark.sql.Column`, str or float, optional - number to calculate logariphm for. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - logariphm of given value. - Returns a column that evaluates to a double. + variance of given column. See Also -------- - :meth:`pyspark.sql.functions.ln` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.var_pop` + :meth:`pyspark.sql.functions.stddev_samp` Examples -------- - Example 1: Specify both base number and the input value - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") - >>> df.select("*", sf.log(2.0, df.value)).show() - +-----+---------------+ - |value|LOG(2.0, value)| - +-----+---------------+ - | 1| 0.0| - | 2| 1.0| - | 4| 2.0| - +-----+---------------+ - - Example 2: Return NULL for invalid input values - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (0), (-1), (NULL) AS t(value)") - >>> df.select("*", sf.log(3.0, df.value)).show() - +-----+------------------+ - |value| LOG(3.0, value)| - +-----+------------------+ - | 1| 0.0| - | 2|0.6309297535714...| - | 0| NULL| - | -1| NULL| - | NULL| NULL| - +-----+------------------+ - - Example 3: Specify only the input value (Natural logarithm) - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") - >>> df.select("*", sf.log(df.value)).show() - +-----+------------------+ - |value| ln(value)| - +-----+------------------+ - | 1| 0.0| - | 2|0.6931471805599...| - | 4|1.3862943611198...| - +-----+------------------+ + >>> df = spark.range(6) + >>> df.select(sf.var_samp(df.id)).show() + +------------+ + |var_samp(id)| + +------------+ + | 3.5| + +------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if arg2 is None: - return _invoke_function_over_columns("log", cast("ColumnOrName", arg1)) - else: - return _invoke_function("log", _enum_to_value(arg1), _to_java_column(arg2)) + return _invoke_function_over_columns("var_samp", col) @_try_remote_functions -def ln(col: "ColumnOrName") -> Column: - """Returns the natural logarithm of the argument. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - a column to calculate logariphm for. - A column that evaluates to a double. - - Returns - ------- - :class:`~pyspark.sql.Column` - natural logarithm of given value. - Returns a column that evaluates to a double. - - See Also - -------- - :meth:`pyspark.sql.functions.log` - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> spark.range(10).select("*", sf.ln('id')).show() - +---+------------------+ - | id| ln(id)| - +---+------------------+ - | 0| NULL| - | 1| 0.0| - | 2|0.6931471805599...| - | 3|1.0986122886681...| - | 4|1.3862943611198...| - | 5|1.6094379124341...| - | 6| 1.791759469228...| - | 7|1.9459101490553...| - | 8|2.0794415416798...| - | 9|2.1972245773362...| - +---+------------------+ +def var_pop(col: "ColumnOrName") -> Column: """ - return _invoke_function_over_columns("ln", col) - - -@_try_remote_functions -def log2(col: "ColumnOrName") -> Column: - """Returns the base-2 logarithm of the argument. + Aggregate function: returns the population variance of the values in a group. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column to calculate logariphm for. - A column that evaluates to a double. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - logariphm of given value. - Returns a column that evaluates to a double. + variance of given column. + + See Also + -------- + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.var_samp` + :meth:`pyspark.sql.functions.stddev_pop` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(10).select("*", sf.log2('id')).show() - +---+------------------+ - | id| LOG2(id)| - +---+------------------+ - | 0| NULL| - | 1| 0.0| - | 2| 1.0| - | 3| 1.584962500721...| - | 4| 2.0| - | 5| 2.321928094887...| - | 6| 2.584962500721...| - | 7| 2.807354922057...| - | 8| 3.0| - | 9|3.1699250014423...| - +---+------------------+ + >>> df = spark.range(6) + >>> df.select(sf.var_pop(df.id)).show() + +------------------+ + | var_pop(id)| + +------------------+ + |2.9166666666666...| + +------------------+ """ - return _invoke_function_over_columns("log2", col) + return _invoke_function_over_columns("var_pop", col) @_try_remote_functions -def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column: +def regr_avgx(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Convert a number in a string column from one base to another. - - .. versionadded:: 1.5.0 + Aggregate function: returns the average of the independent variable for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column to convert base for. - A column that evaluates to a string. - fromBase: int - from base number. - A column that evaluates to an integer. - toBase: int - to base number. - A column that evaluates to an integer. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - logariphm of given value. - Returns a column that evaluates to a string. + the average of the independent variable for non-null pairs in a group. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("010101",), ( "101",), ("001",)], ['n']) - >>> df.select("*", sf.conv(df.n, 2, 16)).show() - +------+--------------+ - | n|conv(n, 2, 16)| - +------+--------------+ - |010101| 15| - | 101| 5| - | 001| 1| - +------+--------------+ - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: All pairs are non-null - return _invoke_function( - "conv", _to_java_column(col), _enum_to_value(fromBase), _enum_to_value(toBase) - ) + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | 2.75| 2.75| + +---------------+------+ + Example 2: All pairs' x values are null -@_try_remote_functions -def factorial(col: "ColumnOrName") -> Column: - """ - Computes the factorial of the given value. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | NULL| NULL| + +---------------+------+ - .. versionadded:: 1.5.0 + Example 3: All pairs' y values are null - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | NULL| 1.0| + +---------------+------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - a column to calculate factorial for. - A column that evaluates to an integer. + Example 4: Some pairs' x values are null - Returns - ------- - :class:`~pyspark.sql.Column` - factorial of given value. - Returns a column that evaluates to a long. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | 3.0| 3.0| + +---------------+------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> spark.range(10).select("*", sf.factorial('id')).show() - +---+-------------+ - | id|factorial(id)| - +---+-------------+ - | 0| 1| - | 1| 1| - | 2| 2| - | 3| 6| - | 4| 24| - | 5| 120| - | 6| 720| - | 7| 5040| - | 8| 40320| - | 9| 362880| - +---+-------------+ + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() + +---------------+------+ + |regr_avgx(y, x)|avg(x)| + +---------------+------+ + | 3.0| 3.0| + +---------------+------+ """ - return _invoke_function_over_columns("factorial", col) + return _invoke_function_over_columns("regr_avgx", y, x) @_try_remote_functions -def bin(col: "ColumnOrName") -> Column: - """Returns the string representation of the binary value of the given column. - - .. versionadded:: 1.5.0 +def regr_avgy(y: "ColumnOrName", x: "ColumnOrName") -> Column: + """ + Aggregate function: returns the average of the dependent variable for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a long. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - binary representation of given value as string. - Returns a column that evaluates to a string. + the average of the dependent variable for non-null pairs in a group. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- + Example 1: All pairs are non-null + >>> import pyspark.sql.functions as sf - >>> spark.range(10).select("*", sf.bin("id")).show() - +---+-------+ - | id|bin(id)| - +---+-------+ - | 0| 0| - | 1| 1| - | 2| 10| - | 3| 11| - | 4| 100| - | 5| 101| - | 6| 110| - | 7| 111| - | 8| 1000| - | 9| 1001| - +---+-------+ - """ - return _invoke_function_over_columns("bin", col) + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------+ + |regr_avgy(y, x)|avg(y)| + +---------------+------+ + | 1.75| 1.75| + +---------------+------+ + Example 2: All pairs' x values are null -@_try_remote_functions -def hex(col: "ColumnOrName") -> Column: - """Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`, - :class:`pyspark.sql.types.BinaryType`, :class:`pyspark.sql.types.IntegerType` or - :class:`pyspark.sql.types.LongType`. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------+ + |regr_avgy(y, x)|avg(y)| + +---------------+------+ + | NULL| 1.0| + +---------------+------+ - .. versionadded:: 1.5.0 + Example 3: All pairs' y values are null - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------+ + |regr_avgy(y, x)|avg(y)| + +---------------+------+ + | NULL| NULL| + +---------------+------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a long, binary, or string. + Example 4: Some pairs' x values are null - See Also - -------- - :meth:`pyspark.sql.functions.unhex` + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +------------------+------+ + | regr_avgy(y, x)|avg(y)| + +------------------+------+ + |1.6666666666666...| 1.75| + +------------------+------+ - Returns - ------- - :class:`~pyspark.sql.Column` - hexadecimal representation of given value as string. - Returns a column that evaluates to a string. + Example 5: Some pairs' x or y values are null - Examples - -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC', 3)], ['a', 'b']) - >>> df.select('*', sf.hex('a'), sf.hex(df.b)).show() - +---+---+------+------+ - | a| b|hex(a)|hex(b)| - +---+---+------+------+ - |ABC| 3|414243| 3| - +---+---+------+------+ + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() + +---------------+------------------+ + |regr_avgy(y, x)| avg(y)| + +---------------+------------------+ + | 1.5|1.6666666666666...| + +---------------+------------------+ """ - return _invoke_function_over_columns("hex", col) + return _invoke_function_over_columns("regr_avgy", y, x) @_try_remote_functions -def unhex(col: "ColumnOrName") -> Column: - """Inverse of hex. Interprets each pair of characters as a hexadecimal number - and converts to the byte representation of number. - - .. versionadded:: 1.5.0 +def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: + """ + Aggregate function: returns the number of non-null number pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. See Also -------- - :meth:`pyspark.sql.functions.hex` + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Returns ------- :class:`~pyspark.sql.Column` - byte representation of the given hexadecimal value. - Returns a column that evaluates to a binary. + the number of non-null number pairs in a group. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('414243',)], ['a']) - >>> df.select('*', sf.unhex('a')).show() - +------+----------+ - | a| unhex(a)| - +------+----------+ - |414243|[41 42 43]| - +------+----------+ - """ - return _invoke_function_over_columns("unhex", col) + Example 1: All pairs are non-null + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 4| 4| + +----------------+--------+ -@_try_remote_functions -def uniform( - min: Union[Column, int, float], - max: Union[Column, int, float], - seed: Optional[Union[Column, int]] = None, -) -> Column: - """Returns a random value with independent and identically distributed (i.i.d.) values with the - specified range of numbers. The random seed is optional. The provided numbers specifying the - minimum and maximum values of the range must be constant. If both of these numbers are integers, - then the result will also be an integer. Otherwise if one or both of these are floating-point - numbers, then the result will also be a floating-point number. + Example 2: All pairs' x values are null - .. versionadded:: 4.0.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 0| 1| + +----------------+--------+ - Parameters - ---------- - min : :class:`~pyspark.sql.Column`, int, or float - Minimum value in the range. - A column that evaluates to a numeric. Must be a constant. - max : :class:`~pyspark.sql.Column`, int, or float - Maximum value in the range. - A column that evaluates to a numeric. Must be a constant. - seed : :class:`~pyspark.sql.Column` or int - Optional random number seed to use. - A column that evaluates to an integer or long. Must be a constant. + Example 3: All pairs' y values are null - Returns - ------- - :class:`~pyspark.sql.Column` - The generated random number within the specified range. - Returns a column of the same type as the input. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 0| 1| + +----------------+--------+ - See Also - -------- - :meth:`pyspark.sql.functions.rand` - :meth:`pyspark.sql.functions.randn` + Example 4: Some pairs' x values are null - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.range(0, 10, 1, 1).select(sf.uniform(5, 105, 3)).show() - +------------------+ - |uniform(5, 105, 3)| - +------------------+ - | 30| - | 71| - | 99| - | 77| - | 16| - | 25| - | 89| - | 80| - | 51| - | 83| - +------------------+ - """ - min = _enum_to_value(min) - min = lit(min) - max = _enum_to_value(max) - max = lit(max) - if seed is None: - return _invoke_function_over_columns("uniform", min, max) - else: - seed = _enum_to_value(seed) - seed = lit(seed) - return _invoke_function_over_columns("uniform", min, max, seed) + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 3| 4| + +----------------+--------+ + Example 5: Some pairs' x or y values are null -# ---------------------- String Functions ---------------------- + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") + >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() + +----------------+--------+ + |regr_count(y, x)|count(0)| + +----------------+--------+ + | 2| 4| + +----------------+--------+ + """ + return _invoke_function_over_columns("regr_count", y, x) @_try_remote_functions -def upper(col: "ColumnOrName") -> Column: +def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Converts a string expression to upper case. - - .. versionadded:: 1.5.0 + Aggregate function: returns the intercept of the univariate linear regression line + for non-null pairs in a group, where `y` is the dependent variable and + `x` is the independent variable. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - upper case values. - Returns a column that evaluates to a string. + the intercept of the univariate linear regression line for non-null pairs in a group. See Also -------- - :meth:`pyspark.sql.functions.lower` - :meth:`pyspark.sql.functions.ucase` + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.upper("value")).show() - +----------+------------+ - | value|upper(value)| - +----------+------------+ - | Spark| SPARK| - | PySpark| PYSPARK| - |Pandas API| PANDAS API| - +----------+------------+ - """ - return _invoke_function_over_columns("upper", col) + Example 1: All pairs are non-null + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | 0.0| + +--------------------+ -@_try_remote_functions -def lower(col: "ColumnOrName") -> Column: - """ - Converts a string expression to lower case. + Example 2: All pairs' x values are null - .. versionadded:: 1.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | NULL| + +--------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: All pairs' y values are null - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | NULL| + +--------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - lower case values. - Returns a column that evaluates to a string. + Example 4: Some pairs' x values are null - See Also - -------- - :meth:`pyspark.sql.functions.upper` - :meth:`pyspark.sql.functions.lcase` + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | 0.0| + +--------------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.lower("value")).show() - +----------+------------+ - | value|lower(value)| - +----------+------------+ - | Spark| spark| - | PySpark| pyspark| - |Pandas API| pandas api| - +----------+------------+ + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_intercept("y", "x")).show() + +--------------------+ + |regr_intercept(y, x)| + +--------------------+ + | 0.0| + +--------------------+ """ - return _invoke_function_over_columns("lower", col) + return _invoke_function_over_columns("regr_intercept", y, x) @_try_remote_functions -def ascii(col: "ColumnOrName") -> Column: +def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Computes the numeric value of the first character of the string column. - - .. versionadded:: 1.5.0 + Aggregate function: returns the coefficient of determination for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - numeric value. - Returns a column that evaluates to an integer. + the coefficient of determination for non-null pairs in a group. + + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.ascii("value")).show() - +----------+------------+ - | value|ascii(value)| - +----------+------------+ - | Spark| 83| - | PySpark| 80| - |Pandas API| 80| - +----------+------------+ - """ - return _invoke_function_over_columns("ascii", col) + Example 1: All pairs are non-null + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | 1.0| + +-------------+ -@_try_remote_functions -def base64(col: "ColumnOrName") -> Column: - """ - Computes the BASE64 encoding of a binary column and returns it as a string column. + Example 2: All pairs' x values are null - .. versionadded:: 1.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | NULL| + +-------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: All pairs' y values are null - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a binary. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | NULL| + +-------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - BASE64 encoding of string value. - Returns a column that evaluates to a string. + Example 4: Some pairs' x values are null - See Also - -------- - :meth:`pyspark.sql.functions.unbase64` - :meth:`pyspark.sql.functions.to_base32` + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | 1.0| + +-------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") - >>> df.select("*", sf.base64("value")).show() - +----------+----------------+ - | value| base64(value)| - +----------+----------------+ - | Spark| U3Bhcms=| - | PySpark| UHlTcGFyaw==| - |Pandas API|UGFuZGFzIEFQSQ==| - +----------+----------------+ + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_r2("y", "x")).show() + +-------------+ + |regr_r2(y, x)| + +-------------+ + | 1.0| + +-------------+ """ - return _invoke_function_over_columns("base64", col) + return _invoke_function_over_columns("regr_r2", y, x) @_try_remote_functions -def to_base32(col: "ColumnOrName") -> Column: +def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a - string column. + Aggregate function: returns the slope of the linear regression line for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a binary. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - BASE32 encoding of the binary value. - Returns a column that evaluates to a string. + the slope of the linear regression line for non-null pairs in a group. See Also -------- - :meth:`pyspark.sql.functions.from_base32` - :meth:`pyspark.sql.functions.base64` + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- + Example 1: All pairs are non-null + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(b"foobar",)], ["value"]) - >>> df.select(sf.to_base32("value").alias("r")).collect() - [Row(r='MZXW6YTBOI======')] - """ - return _invoke_function_over_columns("to_base32", col) + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | 1.0| + +----------------+ + Example 2: All pairs' x values are null -@_try_remote_functions -def unbase64(col: "ColumnOrName") -> Column: - """ - Decodes a BASE64 encoded string column and returns it as a binary column. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | NULL| + +----------------+ - .. versionadded:: 1.5.0 + Example 3: All pairs' y values are null - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | NULL| + +----------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + Example 4: Some pairs' x values are null - Returns - ------- - :class:`~pyspark.sql.Column` - decoded binary value. - Returns a column that evaluates to a binary. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | 1.0| + +----------------+ - See Also - -------- - :meth:`pyspark.sql.functions.base64` - :meth:`pyspark.sql.functions.from_base32` + Example 5: Some pairs' x or y values are null - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["U3Bhcms=", "UHlTcGFyaw==", "UGFuZGFzIEFQSQ=="], "STRING") - >>> df.select("*", sf.unbase64("value")).show(truncate=False) - +----------------+-------------------------------+ - |value |unbase64(value) | - +----------------+-------------------------------+ - |U3Bhcms= |[53 70 61 72 6B] | - |UHlTcGFyaw== |[50 79 53 70 61 72 6B] | - |UGFuZGFzIEFQSQ==|[50 61 6E 64 61 73 20 41 50 49]| - +----------------+-------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_slope("y", "x")).show() + +----------------+ + |regr_slope(y, x)| + +----------------+ + | 1.0| + +----------------+ """ - return _invoke_function_over_columns("unbase64", col) + return _invoke_function_over_columns("regr_slope", y, x) @_try_remote_functions -def from_base32(col: "ColumnOrName") -> Column: +def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary - column. + Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - decoded binary value. - Returns a column that evaluates to a binary. + REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group. See Also -------- - :meth:`pyspark.sql.functions.to_base32` - :meth:`pyspark.sql.functions.unbase64` + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_sxy` + :meth:`pyspark.sql.functions.regr_syy` Examples -------- + Example 1: All pairs are non-null + >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("MZXW6YTBOI======",)], ["value"]) - >>> df.select(sf.from_base32("value").alias("r")).collect() - [Row(r=b'foobar')] + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | 5.0| + +--------------+ + + Example 2: All pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 3: All pairs' y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +-----------------+ + | regr_sxx(y, x)| + +-----------------+ + |4.666666666666...| + +-----------------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxx("y", "x")).show() + +--------------+ + |regr_sxx(y, x)| + +--------------+ + | 4.5| + +--------------+ """ - return _invoke_function_over_columns("from_base32", col) + return _invoke_function_over_columns("regr_sxx", y, x) @_try_remote_functions -def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: +def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Trim the spaces from left end for the specified string value. - - .. versionadded:: 1.5.0 + Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space. - A column that evaluates to a string. + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. - .. versionadded:: 4.0.0 + See Also + -------- + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_syy` Returns ------- :class:`~pyspark.sql.Column` - left trimmed values. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.trim` - :meth:`pyspark.sql.functions.rtrim` + REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group. Examples -------- - Example 1: Trim the spaces + Example 1: All pairs are non-null - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") - >>> df.select("*", sf.ltrim("value")).show() - +--------+------------+ - | value|ltrim(value)| - +--------+------------+ - | Spark| Spark| - | Spark | Spark | - | Spark| Spark| - +--------+------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | 5.0| + +--------------+ - Example 2: Trim specified characters + Example 2: All pairs' x values are null - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") - >>> df.select("*", sf.ltrim("value", sf.lit("*"))).show() - +--------+--------------------------+ - | value|TRIM(LEADING * FROM value)| - +--------+--------------------------+ - |***Spark| Spark| - | Spark**| Spark**| - | *Spark| Spark| - +--------+--------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | NULL| + +--------------+ - Example 3: Trim a column containing different characters + Example 3: All pairs' y values are null - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) - >>> df.select("*", sf.ltrim("value", "t")).show() - +--------+---+--------------------------+ - | value| t|TRIM(LEADING t FROM value)| - +--------+---+--------------------------+ - |**Spark*| *| Spark*| - |==Spark=| =| Spark=| - +--------+---+--------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +-----------------+ + | regr_sxy(y, x)| + +-----------------+ + |4.666666666666...| + +-----------------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_sxy("y", "x")).show() + +--------------+ + |regr_sxy(y, x)| + +--------------+ + | 4.5| + +--------------+ """ - if trim is not None: - return _invoke_function_over_columns("ltrim", col, trim) - else: - return _invoke_function_over_columns("ltrim", col) + return _invoke_function_over_columns("regr_sxy", y, x) @_try_remote_functions -def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: +def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: """ - Trim the spaces from right end for the specified string value. - - .. versionadded:: 1.5.0 + Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs + in a group, where `y` is the dependent variable and `x` is the independent variable. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space. - A column that evaluates to a string. - - .. versionadded:: 4.0.0 + y : :class:`~pyspark.sql.Column` or column name + the dependent variable. + A column that evaluates to a numeric. + x : :class:`~pyspark.sql.Column` or column name + the independent variable. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - right trimmed values. - Returns a column that evaluates to a string. + REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group. See Also -------- - :meth:`pyspark.sql.functions.trim` - :meth:`pyspark.sql.functions.ltrim` + :meth:`pyspark.sql.functions.regr_avgx` + :meth:`pyspark.sql.functions.regr_avgy` + :meth:`pyspark.sql.functions.regr_count` + :meth:`pyspark.sql.functions.regr_intercept` + :meth:`pyspark.sql.functions.regr_r2` + :meth:`pyspark.sql.functions.regr_slope` + :meth:`pyspark.sql.functions.regr_sxy` Examples -------- - Example 1: Trim the spaces + Example 1: All pairs are non-null - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") - >>> df.select("*", sf.rtrim("value")).show() - +--------+------------+ - | value|rtrim(value)| - +--------+------------+ - | Spark| Spark| - | Spark | Spark| - | Spark| Spark| - +--------+------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | 5.0| + +--------------+ - Example 2: Trim specified characters + Example 2: All pairs' x values are null - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") - >>> df.select("*", sf.rtrim("value", sf.lit("*"))).show() - +--------+---------------------------+ - | value|TRIM(TRAILING * FROM value)| - +--------+---------------------------+ - |***Spark| ***Spark| - | Spark**| Spark| - | *Spark| *Spark| - +--------+---------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | NULL| + +--------------+ - Example 3: Trim a column containing different characters + Example 3: All pairs' y values are null - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) - >>> df.select("*", sf.rtrim("value", "t")).show() - +--------+---+---------------------------+ - | value| t|TRIM(TRAILING t FROM value)| - +--------+---+---------------------------+ - |**Spark*| *| **Spark| - |==Spark=| =| ==Spark| - +--------+---+---------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | NULL| + +--------------+ + + Example 4: Some pairs' x values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +-----------------+ + | regr_syy(y, x)| + +-----------------+ + |4.666666666666...| + +-----------------+ + + Example 5: Some pairs' x or y values are null + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") + >>> df.select(sf.regr_syy("y", "x")).show() + +--------------+ + |regr_syy(y, x)| + +--------------+ + | 4.5| + +--------------+ """ - if trim is not None: - return _invoke_function_over_columns("rtrim", col, trim) - else: - return _invoke_function_over_columns("rtrim", col) + return _invoke_function_over_columns("regr_syy", y, x) @_try_remote_functions -def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: +def every(col: "ColumnOrName") -> Column: """ - Trim the spaces from both ends for the specified string column. - - .. versionadded:: 1.5.0 + Aggregate function: returns true if all values of `col` are true. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or column name, optional - The trim string characters to trim, the default value is a single space. - A column that evaluates to a string. + column to check if all values are true. + A column that evaluates to a boolean. - .. versionadded:: 4.0.0 + See Also + -------- + :meth:`pyspark.sql.functions.some` Returns ------- :class:`~pyspark.sql.Column` - trimmed values from both sides. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.ltrim` - :meth:`pyspark.sql.functions.rtrim` + true if all values of `col` are true, false otherwise. Examples -------- - Example 1: Trim the spaces - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") - >>> df.select("*", sf.trim("value")).show() - +--------+-----------+ - | value|trim(value)| - +--------+-----------+ - | Spark| Spark| - | Spark | Spark| - | Spark| Spark| - +--------+-----------+ - - Example 2: Trim specified characters - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") - >>> df.select("*", sf.trim("value", sf.lit("*"))).show() - +--------+-----------------------+ - | value|TRIM(BOTH * FROM value)| - +--------+-----------------------+ - |***Spark| Spark| - | Spark**| Spark| - | *Spark| Spark| - +--------+-----------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [True], [True]], ["flag"] + ... ).select(sf.every("flag")).show() + +-----------+ + |every(flag)| + +-----------+ + | true| + +-----------+ - Example 3: Trim a column containing different characters + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [False], [True]], ["flag"] + ... ).select(sf.every("flag")).show() + +-----------+ + |every(flag)| + +-----------+ + | false| + +-----------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) - >>> df.select("*", sf.trim("value", "t")).show() - +--------+---+-----------------------+ - | value| t|TRIM(BOTH t FROM value)| - +--------+---+-----------------------+ - |**Spark*| *| Spark| - |==Spark=| =| Spark| - +--------+---+-----------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[False], [False], [False]], ["flag"] + ... ).select(sf.every("flag")).show() + +-----------+ + |every(flag)| + +-----------+ + | false| + +-----------+ """ - if trim is not None: - return _invoke_function_over_columns("trim", col, trim) - else: - return _invoke_function_over_columns("trim", col) + return _invoke_function_over_columns("every", col) @_try_remote_functions -def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: +def bool_and(col: "ColumnOrName") -> Column: """ - Concatenates multiple input string columns together into a single string column, - using the given separator. - - .. versionadded:: 1.5.0 + Aggregate function: returns true if all values of `col` are true. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - sep : literal string - words separator. - A column that evaluates to a string. - cols : :class:`~pyspark.sql.Column` or column name - list of columns to work on. - Each a column that evaluates to a string or an array of strings. + col : :class:`~pyspark.sql.Column` or column name + column to check if all values are true. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - string of concatenated words. - Returns a column that evaluates to a string. + true if all values of `col` are true, false otherwise. See Also -------- - :meth:`pyspark.sql.functions.concat` + :meth:`pyspark.sql.functions.bool_or` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd", "123")], ["s", "d"]) - >>> df.select("*", sf.concat_ws("-", df.s, "d", sf.lit("xyz"))).show() - +----+---+-----------------------+ - | s| d|concat_ws(-, s, d, xyz)| - +----+---+-----------------------+ - |abcd|123| abcd-123-xyz| - +----+---+-----------------------+ - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) + >>> df.select(sf.bool_and("flag")).show() + +--------------+ + |bool_and(flag)| + +--------------+ + | true| + +--------------+ - sc = _get_active_spark_context() - return _invoke_function("concat_ws", _enum_to_value(sep), _to_seq(sc, cols, _to_java_column)) + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) + >>> df.select(sf.bool_and("flag")).show() + +--------------+ + |bool_and(flag)| + +--------------+ + | false| + +--------------+ + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) + >>> df.select(sf.bool_and("flag")).show() + +--------------+ + |bool_and(flag)| + +--------------+ + | false| + +--------------+ + """ + return _invoke_function_over_columns("bool_and", col) @_try_remote_functions -def decode(col: "ColumnOrName", charset: str) -> Column: +def some(col: "ColumnOrName") -> Column: """ - Computes the first argument into a string from a binary using the provided character set - (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). - - .. versionadded:: 1.5.0 + Aggregate function: returns true if at least one value of `col` is true. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to work on. - charset : literal string - charset to use to decode to. + column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + true if at least one value of `col` is true, false otherwise. See Also -------- - :meth:`pyspark.sql.functions.encode` + :meth:`pyspark.sql.functions.every` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b"\x61\x62\x63\x64",)], ["a"]) - >>> df.select("*", sf.decode("a", "UTF-8")).show() - +-------------+----------------+ - | a|decode(a, UTF-8)| - +-------------+----------------+ - |[61 62 63 64]| abcd| - +-------------+----------------+ - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("decode", _to_java_column(col), _enum_to_value(charset)) + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [True], [True]], ["flag"] + ... ).select(sf.some("flag")).show() + +----------+ + |some(flag)| + +----------+ + | true| + +----------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[True], [False], [True]], ["flag"] + ... ).select(sf.some("flag")).show() + +----------+ + |some(flag)| + +----------+ + | true| + +----------+ -@_try_remote_functions -def encode(col: "ColumnOrName", charset: str) -> Column: + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [[False], [False], [False]], ["flag"] + ... ).select(sf.some("flag")).show() + +----------+ + |some(flag)| + +----------+ + | false| + +----------+ """ - Computes the first argument into a binary from a string using the provided character set - (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). + return _invoke_function_over_columns("some", col) - .. versionadded:: 1.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def bool_or(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns true if at least one value of `col` is true. + + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - charset : literal string - charset to use to encode. - A column that evaluates to a string. + column to check if at least one value is true. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. - Returns a column that evaluates to a binary. + true if at least one value of `col` is true, false otherwise. See Also -------- - :meth:`pyspark.sql.functions.decode` + :meth:`pyspark.sql.functions.bool_and` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd",)], ["c"]) - >>> df.select("*", sf.encode("c", "UTF-8")).show() - +----+----------------+ - | c|encode(c, UTF-8)| - +----+----------------+ - |abcd| [61 62 63 64]| - +----+----------------+ + >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) + >>> df.select(bool_or("flag")).show() + +-------------+ + |bool_or(flag)| + +-------------+ + | true| + +-------------+ + >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) + >>> df.select(bool_or("flag")).show() + +-------------+ + |bool_or(flag)| + +-------------+ + | true| + +-------------+ + >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) + >>> df.select(bool_or("flag")).show() + +-------------+ + |bool_or(flag)| + +-------------+ + | false| + +-------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("encode", _to_java_column(col), _enum_to_value(charset)) + return _invoke_function_over_columns("bool_or", col) @_try_remote_functions -def is_valid_utf8(str: "ColumnOrName") -> Column: +def bit_and(col: "ColumnOrName") -> Column: """ - Returns true if the input is a valid UTF-8 string, otherwise returns false. + Aggregate function: returns the bitwise AND of all non-null input values, or null if none. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` - whether the input string is a valid UTF-8 string. - Returns a column that evaluates to a boolean. + the bitwise AND of all non-null input values, or null if none. See Also -------- - :meth:`pyspark.sql.functions.make_valid_utf8` - :meth:`pyspark.sql.functions.validate_utf8` - :meth:`pyspark.sql.functions.try_validate_utf8` + :meth:`pyspark.sql.functions.bit_or` + :meth:`pyspark.sql.functions.bit_xor` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.is_valid_utf8(sf.lit("SparkSQL"))).show() - +-----------------------+ - |is_valid_utf8(SparkSQL)| - +-----------------------+ - | true| - +-----------------------+ - """ - return _invoke_function_over_columns("is_valid_utf8", str) + Example 1: Bitwise AND with all non-null values + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | 0| + +----------+ -@_try_remote_functions -def make_valid_utf8(str: "ColumnOrName") -> Column: - """ - Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the - Unicode replacement character (U+FFFD). + Example 2: Bitwise AND with null values - .. versionadded:: 4.0.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | 0| + +----------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + Example 3: Bitwise AND with all null values - Returns - ------- - :class:`~pyspark.sql.Column` - the valid UTF-8 version of the given input string. - Returns a column that evaluates to a string. + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([StructField("c", IntegerType(), True)]) + >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | NULL| + +----------+ - See Also - -------- - :meth:`pyspark.sql.functions.is_valid_utf8` - :meth:`pyspark.sql.functions.validate_utf8` - :meth:`pyspark.sql.functions.try_validate_utf8` + Example 4: Bitwise AND with single input value - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_valid_utf8(sf.lit("SparkSQL"))).show() - +-------------------------+ - |make_valid_utf8(SparkSQL)| - +-------------------------+ - | SparkSQL| - +-------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[5]], ["c"]) + >>> df.select(sf.bit_and("c")).show() + +----------+ + |bit_and(c)| + +----------+ + | 5| + +----------+ """ - return _invoke_function_over_columns("make_valid_utf8", str) + return _invoke_function_over_columns("bit_and", col) @_try_remote_functions -def validate_utf8(str: "ColumnOrName") -> Column: +def bit_or(col: "ColumnOrName") -> Column: """ - Returns the input value if it corresponds to a valid UTF-8 string, or emits an error otherwise. + Aggregate function: returns the bitwise OR of all non-null input values, or null if none. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` - the input string if it is a valid UTF-8 string, error otherwise. - Returns a column that evaluates to a string. + the bitwise OR of all non-null input values, or null if none. See Also -------- - :meth:`pyspark.sql.functions.is_valid_utf8` - :meth:`pyspark.sql.functions.make_valid_utf8` - :meth:`pyspark.sql.functions.try_validate_utf8` + :meth:`pyspark.sql.functions.bit_and` + :meth:`pyspark.sql.functions.bit_xor` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.validate_utf8(sf.lit("SparkSQL"))).show() - +-----------------------+ - |validate_utf8(SparkSQL)| - +-----------------------+ - | SparkSQL| - +-----------------------+ + Example 1: Bitwise OR with all non-null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | 3| + +---------+ + + Example 2: Bitwise OR with some null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | 3| + +---------+ + + Example 3: Bitwise OR with all null values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([StructField("c", IntegerType(), True)]) + >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | NULL| + +---------+ + + Example 4: Bitwise OR with single input value + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[5]], ["c"]) + >>> df.select(sf.bit_or("c")).show() + +---------+ + |bit_or(c)| + +---------+ + | 5| + +---------+ """ - return _invoke_function_over_columns("validate_utf8", str) + return _invoke_function_over_columns("bit_or", col) @_try_remote_functions -def try_validate_utf8(str: "ColumnOrName") -> Column: +def bit_xor(col: "ColumnOrName") -> Column: """ - Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. + Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of strings, each representing a UTF-8 byte sequence. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to an integral. Returns ------- :class:`~pyspark.sql.Column` - the input string if it is a valid UTF-8 string, null otherwise. - Returns a column that evaluates to a string. + the bitwise XOR of all non-null input values, or null if none. See Also -------- - :meth:`pyspark.sql.functions.is_valid_utf8` - :meth:`pyspark.sql.functions.make_valid_utf8` - :meth:`pyspark.sql.functions.validate_utf8` + :meth:`pyspark.sql.functions.bit_and` + :meth:`pyspark.sql.functions.bit_or` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.try_validate_utf8(sf.lit("SparkSQL"))).show() - +---------------------------+ - |try_validate_utf8(SparkSQL)| - +---------------------------+ - | SparkSQL| - +---------------------------+ - """ - return _invoke_function_over_columns("try_validate_utf8", str) + Example 1: Bitwise XOR with all non-null values + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | 2| + +----------+ -@_try_remote_functions -def normalize(str: "ColumnOrName", form: Optional["ColumnOrName"] = None) -> Column: - """ - Returns the Unicode normalization of ``str`` using the given normalization ``form``, as - defined by Unicode Standard Annex #15. Normalization is backed by Spark's bundled ICU4J - library rather than the JVM's own Unicode data, so results are stable across JVM vendors - and versions. + Example 2: Bitwise XOR with some null values - .. versionadded:: 4.4.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | 3| + +----------+ - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - the input string to normalize. - form : :class:`~pyspark.sql.Column` or column name, optional - the normalization form, one of 'NFC', 'NFD', 'NFKC', 'NFKD' (case-insensitive). - If omitted, 'NFC' is used. + Example 3: Bitwise XOR with all null values - Returns - ------- - :class:`~pyspark.sql.Column` - the normalized string. + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([StructField("c", IntegerType(), True)]) + >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | NULL| + +----------+ - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("\ufb01",)], ["s"]) - >>> df.select(sf.normalize(df.s, sf.lit("NFKC"))).show() - +------------------+ - |normalize(s, NFKC)| - +------------------+ - | fi| - +------------------+ + Example 4: Bitwise XOR with single input value + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[5]], ["c"]) + >>> df.select(sf.bit_xor("c")).show() + +----------+ + |bit_xor(c)| + +----------+ + | 5| + +----------+ """ - if form is None: - return _invoke_function_over_columns("normalize", str) - else: - return _invoke_function_over_columns("normalize", str, form) + return _invoke_function_over_columns("bit_xor", col) @_try_remote_functions -def format_number(col: "ColumnOrName", d: int) -> Column: +def skewness(col: "ColumnOrName") -> Column: """ - Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places - with HALF_EVEN round mode, and returns the result as a string. + Aggregate function: returns the skewness of the values in a group. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -5920,1696 +6021,1363 @@ def format_number(col: "ColumnOrName", d: int) -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - the column name of the numeric value to be formatted. + target column to compute on. A column that evaluates to a numeric. - d : int - the N decimal places. - A column that evaluates to an integer. + + See Also + -------- + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.kurtosis` Returns ------- :class:`~pyspark.sql.Column` - the column of formatted results. - Returns a column that evaluates to a string. + skewness of given column. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(5,)], ["a"]) - >>> df.select("*", sf.format_number("a", 4), sf.format_number(df.a, 6)).show() - +---+-------------------+-------------------+ - | a|format_number(a, 4)|format_number(a, 6)| - +---+-------------------+-------------------+ - | 5| 5.0000| 5.000000| - +---+-------------------+-------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.skewness(df.c)).show() + +------------------+ + | skewness(c)| + +------------------+ + |0.7071067811865...| + +------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("format_number", _to_java_column(col), _enum_to_value(d)) + return _invoke_function_over_columns("skewness", col) @_try_remote_functions -def format_string(format: str, *cols: "ColumnOrName") -> Column: +def kurtosis(col: "ColumnOrName") -> Column: """ - Formats the arguments in printf-style and returns the result as a string column. + Aggregate function: returns the kurtosis of the values in a group. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - format : literal string - string that can contain embedded format tags and used as result column's value. - A column that evaluates to a string. - cols : :class:`~pyspark.sql.Column` or column name - column names or :class:`~pyspark.sql.Column`\\s to be used in formatting - Each a column of any type. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - the column of formatted results. - Returns a column that evaluates to a string. + kurtosis of given column. See Also -------- - :meth:`pyspark.sql.functions.printf` + :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.stddev` + :meth:`pyspark.sql.functions.variance` + :meth:`pyspark.sql.functions.skewness` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(5, "hello")], ["a", "b"]) - >>> df.select("*", sf.format_string('%d %s', "a", df.b)).show() - +---+-----+--------------------------+ - | a| b|format_string(%d %s, a, b)| - +---+-----+--------------------------+ - | 5|hello| 5 hello| - +---+-----+--------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.select(sf.kurtosis(df.c)).show() + +-----------+ + |kurtosis(c)| + +-----------+ + | -1.5| + +-----------+ """ - from pyspark.sql.classic.column import _to_java_column, _to_seq - - sc = _get_active_spark_context() - return _invoke_function( - "format_string", _enum_to_value(format), _to_seq(sc, cols, _to_java_column) - ) + return _invoke_function_over_columns("kurtosis", col) @_try_remote_functions -def instr( - str: "ColumnOrName", - substr: Union[Column, str], - start: Optional[Union[Column, int]] = None, - occurrence: Optional[Union[Column, int]] = None, -) -> Column: +def collect_list(col: "ColumnOrName") -> Column: """ - Locate the position of the specified occurrence of substr column in the given string. - Returns null if either of the arguments are null. + Aggregate function: Collects the values from a column into a list, + maintaining duplicates, and returns this list of objects. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.3.0 - Supports optional `start` and `occurrence` parameters. - - Notes - ----- - The position is not zero based, but 1 based index. Returns 0 if substr - could not be found in str. - Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - substr : :class:`~pyspark.sql.Column` or literal string - substring to look for. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The target column on which the function is computed. - .. versionchanged:: 4.0.0 - `substr` now accepts column. - start : int or :class:`~pyspark.sql.Column`, optional - Starting position (1-based, can be negative for backward search). - If not specified, defaults to 1. - A column that evaluates to an integer. - occurrence : int or :class:`~pyspark.sql.Column`, optional - Which occurrence to locate (must be > 0). Defaults to 1. - A column that evaluates to an integer. + See Also + -------- + :meth:`pyspark.sql.functions.array_agg` + :meth:`pyspark.sql.functions.collect_set` Returns ------- :class:`~pyspark.sql.Column` - location of the substring as integer. - Returns a column that evaluates to an integer. + A new Column object representing a list of collected values, with duplicate values included. - See Also - -------- - :meth:`pyspark.sql.functions.locate` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.functions.substring_index` + Notes + ----- + The function is non-deterministic as the order of collected results depends + on the order of the rows, which possibly becomes non-deterministic after shuffle operations. Examples -------- - Example 1: Using a literal string as the 'substring' + Example 1: Collect values from a DataFrame and sort the result in ascending order >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr(df.s, "b")).show() - +----+-----------+ - | s|instr(s, b)| - +----+-----------+ - |abcd| 2| - | xyz| 0| - +----+-----------+ + >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) + >>> df.select(sf.sort_array(sf.collect_list('value')).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [1, 2, 2]| + +-----------+ - Example 2: Using a Column 'substring' + Example 2: Collect values from a DataFrame and sort the result in descending order >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr("s", sf.lit("abc").substr(0, 2))).show() - +----+---------------------------+ - | s|instr(s, substr(abc, 0, 2))| - +----+---------------------------+ - |abcd| 1| - | xyz| 0| - +----+---------------------------+ - - Example 3: Using start and occurrence parameters - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr("s", "b", 1, 2)).show() - +-----+-----------------+ - | s|instr(s, b, 1, 2)| - +-----+-----------------+ - |aabcd| 0| - | xyz| 0| - +-----+-----------------+ + >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) + >>> df.select(sf.sort_array(sf.collect_list('age'), asc=False).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [5, 5, 2]| + +-----------+ - Example 4: Using start parameter + Example 3: Collect values from a DataFrame with multiple columns and sort the result >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) - >>> df.select("*", sf.instr("s", "a", 2)).show() - +-----+-----------------+ - | s|instr(s, a, 2, 1)| - +-----+-----------------+ - |aabcd| 2| - | xyz| 0| - +-----+-----------------+ + >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) + >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_list('id')).alias('sorted_list')) + >>> df.orderBy(sf.desc("name")).show() + +----+-----------+ + |name|sorted_list| + +----+-----------+ + |John| [1, 2]| + | Ana| [3]| + +----+-----------+ """ - if start is None and occurrence is None: - return _invoke_function_over_columns("instr", str, lit(substr)) - elif start is not None and occurrence is None: - start = lit(start) - return _invoke_function_over_columns("instr", str, lit(substr), start) - else: - start = lit(start) if start is not None else lit(1) - occurrence = lit(occurrence) - return _invoke_function_over_columns("instr", str, lit(substr), start, occurrence) + return _invoke_function_over_columns("collect_list", col) @_try_remote_functions -def overlay( - src: "ColumnOrName", - replace: "ColumnOrName", - pos: Union["ColumnOrName", int], - len: Union["ColumnOrName", int] = -1, -) -> Column: +def array_agg(col: "ColumnOrName") -> Column: """ - Overlay the specified portion of `src` with `replace`, - starting from byte position `pos` of `src` and proceeding for `len` bytes. - - .. versionadded:: 3.0.0 + Aggregate function: returns a list of objects with duplicates. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - src : :class:`~pyspark.sql.Column` or column name - the string that will be replaced. - A column that evaluates to a string or binary. - replace : :class:`~pyspark.sql.Column` or column name - the substitution string. - A column that evaluates to a string or binary. - pos : :class:`~pyspark.sql.Column` or column name or int - the starting position in src. - A column that evaluates to an integer. - len : :class:`~pyspark.sql.Column` or column name or int, optional - the number of bytes to replace in src - string by 'replace' defaults to -1, which represents the length of the 'replace' string. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. Returns ------- :class:`~pyspark.sql.Column` - string with replaced values. - Returns a column of the same type as the input. + list of objects with duplicates. + + See Also + -------- + :meth:`pyspark.sql.functions.collect_list` + :meth:`pyspark.sql.functions.collect_set` Examples -------- + Example 1: Using array_agg function on an int column + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("SPARK_SQL", "CORE")], ("x", "y")) - >>> df.select("*", sf.overlay("x", df.y, 7)).show() - +---------+----+--------------------+ - | x| y|overlay(x, y, 7, -1)| - +---------+----+--------------------+ - |SPARK_SQL|CORE| SPARK_CORE| - +---------+----+--------------------+ + >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [1, 1, 2]| + +-----------+ - >>> df.select("*", sf.overlay("x", df.y, 7, 0)).show() - +---------+----+-------------------+ - | x| y|overlay(x, y, 7, 0)| - +---------+----+-------------------+ - |SPARK_SQL|CORE| SPARK_CORESQL| - +---------+----+-------------------+ + Example 2: Using array_agg function on a string column - >>> df.select("*", sf.overlay("x", "y", 7, 2)).show() - +---------+----+-------------------+ - | x| y|overlay(x, y, 7, 2)| - +---------+----+-------------------+ - |SPARK_SQL|CORE| SPARK_COREL| - +---------+----+-------------------+ - """ - pos = _enum_to_value(pos) - if not isinstance(pos, (int, str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column, int or str", - "arg_name": "pos", - "arg_type": type(pos).__name__, - }, - ) - len = _enum_to_value(len) - if len is not None and not isinstance(len, (int, str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column, int or str", - "arg_name": "len", - "arg_type": type(len).__name__, - }, - ) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([["apple"],["apple"],["banana"]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show(truncate=False) + +----------------------+ + |sorted_list | + +----------------------+ + |[apple, apple, banana]| + +----------------------+ - if isinstance(pos, int): - pos = lit(pos) - if isinstance(len, int): - len = lit(len) + Example 3: Using array_agg function on a column with null values - return _invoke_function_over_columns("overlay", src, replace, pos, len) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() + +-----------+ + |sorted_list| + +-----------+ + | [1, 2]| + +-----------+ + + Example 4: Using array_agg function on a column with different data types + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([[1],["apple"],[2]], ["c"]) + >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() + +-------------+ + | sorted_list| + +-------------+ + |[1, 2, apple]| + +-------------+ + """ + return _invoke_function_over_columns("array_agg", col) @_try_remote_functions -def sentences( - string: "ColumnOrName", - language: Optional["ColumnOrName"] = None, - country: Optional["ColumnOrName"] = None, -) -> Column: +def collect_set(col: "ColumnOrName") -> Column: """ - Splits a string into arrays of sentences, where each sentence is an array of words. - The `language` and `country` arguments are optional, - When they are omitted: - 1.If they are both omitted, the `Locale.ROOT - locale(language='', country='')` is used. - The `Locale.ROOT` is regarded as the base locale of all locales, and is used as the - language/country neutral locale for the locale sensitive operations. - 2.If the `country` is omitted, the `locale(language, country='')` is used. - When they are null: - 1.If they are both `null`, the `Locale.US - locale(language='en', country='US')` is used. - 2.If the `language` is null and the `country` is not null, - the `Locale.US - locale(language='en', country='US')` is used. - 3.If the `language` is not null and the `country` is null, the `locale(language)` is used. - 4.If neither is `null`, the `locale(language, country)` is used. + Aggregate function: Collects the values from a column into a set, + eliminating duplicates, and returns this set of objects. - .. versionadded:: 3.2.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.0.0 - Supports `sentences(string, language)`. - Parameters ---------- - string : :class:`~pyspark.sql.Column` or column name - a string to be split. - A column that evaluates to a string. - language : :class:`~pyspark.sql.Column` or column name, optional - a language of the locale. - A column that evaluates to a string. - country : :class:`~pyspark.sql.Column` or column name, optional - a country of the locale. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The target column on which the function is computed. Returns ------- :class:`~pyspark.sql.Column` - arrays of split sentences. - Returns a column that evaluates to an array. + A new Column object representing a set of collected values, duplicates excluded. See Also -------- - :meth:`pyspark.sql.functions.split` - :meth:`pyspark.sql.functions.split_part` + :meth:`pyspark.sql.functions.array_agg` + :meth:`pyspark.sql.functions.collect_list` + + Notes + ----- + This function is non-deterministic as the order of collected results depends + on the order of the rows, which may be non-deterministic after any shuffle operations. Examples -------- + Example 1: Collect values from a DataFrame and sort the result in ascending order + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("This is an example sentence.", )], ["s"]) - >>> df.select("*", sf.sentences(df.s, sf.lit("en"), sf.lit("US"))).show(truncate=False) - +----------------------------+-----------------------------------+ - |s |sentences(s, en, US) | - +----------------------------+-----------------------------------+ - |This is an example sentence.|[[This, is, an, example, sentence]]| - +----------------------------+-----------------------------------+ + >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) + >>> df.select(sf.sort_array(sf.collect_set('value')).alias('sorted_set')).show() + +----------+ + |sorted_set| + +----------+ + | [1, 2]| + +----------+ - >>> df.select("*", sf.sentences(df.s, sf.lit("en"))).show(truncate=False) - +----------------------------+-----------------------------------+ - |s |sentences(s, en, ) | - +----------------------------+-----------------------------------+ - |This is an example sentence.|[[This, is, an, example, sentence]]| - +----------------------------+-----------------------------------+ + Example 2: Collect values from a DataFrame and sort the result in descending order - >>> df.select("*", sf.sentences(df.s)).show(truncate=False) - +----------------------------+-----------------------------------+ - |s |sentences(s, , ) | - +----------------------------+-----------------------------------+ - |This is an example sentence.|[[This, is, an, example, sentence]]| - +----------------------------+-----------------------------------+ - """ - if language is None: - language = lit("") - if country is None: - country = lit("") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) + >>> df.select(sf.sort_array(sf.collect_set('age'), asc=False).alias('sorted_set')).show() + +----------+ + |sorted_set| + +----------+ + | [5, 2]| + +----------+ - return _invoke_function_over_columns("sentences", string, language, country) + Example 3: Collect values from a DataFrame with multiple columns and sort the result + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) + >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_set('id')).alias('sorted_set')) + >>> df.orderBy(sf.desc("name")).show() + +----+----------+ + |name|sorted_set| + +----+----------+ + |John| [1, 2]| + | Ana| [3]| + +----+----------+ + """ + return _invoke_function_over_columns("collect_set", col) @_try_remote_functions -def substring( - str: "ColumnOrName", - pos: Union["ColumnOrName", int], - len: Union["ColumnOrName", int], -) -> Column: +def collect_union(col: "ColumnOrName") -> Column: """ - Substring starts at `pos` and is of length `len` when str is String type or - returns the slice of byte array that starts at `pos` in byte and is of length `len` - when str is Binary type. - - .. versionadded:: 1.5.0 + Aggregate function: given an array-typed column, collects the distinct union of the + elements of the arrays across rows and returns it as an array. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + The aggregation buffer holds only the distinct elements, so its size is bounded by the + element universe rather than by the number of input rows. Null elements are dropped by + default (``IGNORE NULLS``), matching :func:`collect_set`. With ``RESPECT NULLS`` a single + null element is kept, in which case this is equivalent to + ``array_distinct(flatten(collect_list(col)))``. The ``RESPECT NULLS`` clause is only + available through SQL, e.g. ``expr("collect_union(col) RESPECT NULLS")``. - Notes - ----- - The position is not zero based, but 1 based index. + .. versionadded:: 4.3.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string or binary. - pos : :class:`~pyspark.sql.Column` or column name or int - starting position in str. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `pos` now accepts column and column name. - - len : :class:`~pyspark.sql.Column` or column name or int - length of chars. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `len` now accepts column and column name. + col : :class:`~pyspark.sql.Column` or column name + The target array column on which the function is computed. Returns ------- :class:`~pyspark.sql.Column` - substring of given value. - Returns a column of the same type as the input. + A new Column object representing the distinct union of the array elements. See Also -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.locate` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.Column.substr` + :meth:`pyspark.sql.functions.collect_set` + :meth:`pyspark.sql.functions.collect_list` + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.flatten` + + Notes + ----- + This function is non-deterministic as the order of collected results depends + on the order of the rows, which may be non-deterministic after any shuffle operations. Examples -------- - Example 1: Using literal integers as arguments - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('abcd',)], ['s',]) - >>> df.select('*', sf.substring(df.s, 1, 2)).show() - +----+------------------+ - | s|substring(s, 1, 2)| - +----+------------------+ - |abcd| ab| - +----+------------------+ - - Example 2: Using columns as arguments - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) - >>> df.select('*', sf.substring(df.s, 2, df.l)).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, 2, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ - - >>> df.select('*', sf.substring(df.s, df.p, 3)).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, p, 3)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ - - >>> df.select('*', sf.substring(df.s, df.p, df.l)).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, p, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + Example 1: Union the elements of array columns across rows - Example 3: Using column names as arguments + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [([1, 2],), ([2, 3],), ([1],)], ('value',)) + >>> df.select(sf.sort_array(sf.collect_union('value')).alias('u')).show() + +---------+ + | u| + +---------+ + |[1, 2, 3]| + +---------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) - >>> df.select('*', sf.substring(df.s, 2, 'l')).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, 2, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + Example 2: Union per group - >>> df.select('*', sf.substring('s', 'p', 'l')).show() - +-----+---+---+------------------+ - | s| p| l|substring(s, p, l)| - +-----+---+---+------------------+ - |Spark| 2| 3| par| - +-----+---+---+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("a", [1, 2]), ("a", [2, 3]), ("b", [4])], ("k", "value")) + >>> df = df.groupBy("k").agg(sf.sort_array(sf.collect_union('value')).alias('u')) + >>> df.orderBy("k").show() + +---+---------+ + | k| u| + +---+---------+ + | a|[1, 2, 3]| + | b| [4]| + +---+---------+ """ - pos = _enum_to_value(pos) - pos = lit(pos) if isinstance(pos, int) else pos - len = _enum_to_value(len) - len = lit(len) if isinstance(len, int) else len - return _invoke_function_over_columns("substring", str, pos, len) + return _invoke_function_over_columns("collect_union", col) @_try_remote_functions -def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: +def degrees(col: "ColumnOrName") -> Column: """ - Returns the substring from string str before count occurrences of the delimiter delim. - If count is positive, everything the left of the final delimiter (counting from left) is - returned. If count is negative, every to the right of the final delimiter (counting from the - right) is returned. substring_index performs a case-sensitive match when searching for delim. + Converts an angle measured in radians to an approximately equivalent angle + measured in degrees. - .. versionadded:: 1.5.0 + .. versionadded:: 2.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - delim : literal string - delimiter of values. - A column that evaluates to a string. - count : int - number of occurrences. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + angle in radians. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - substring of given value. - Returns a column that evaluates to a string. + angle in degrees, as if computed by `java.lang.Math.toDegrees()` + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.locate` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.Column.substr` + :meth:`pyspark.sql.functions.radians` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a.b.c.d',)], ['s']) - >>> df.select('*', sf.substring_index(df.s, '.', 2)).show() - +-------+------------------------+ - | s|substring_index(s, ., 2)| - +-------+------------------------+ - |a.b.c.d| a.b| - +-------+------------------------+ - - >>> df.select('*', sf.substring_index('s', '.', -3)).show() - +-------+-------------------------+ - | s|substring_index(s, ., -3)| - +-------+-------------------------+ - |a.b.c.d| b.c.d| - +-------+-------------------------+ + >>> spark.sql( + ... "SELECT * FROM VALUES (0.0), (PI()), (PI() / 2), (PI() / 4) AS TAB(value)" + ... ).select("*", sf.degrees("value")).show() + +------------------+--------------+ + | value|DEGREES(value)| + +------------------+--------------+ + | 0.0| 0.0| + | 3.141592653589...| 180.0| + |1.5707963267948...| 90.0| + |0.7853981633974...| 45.0| + +------------------+--------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "substring_index", _to_java_column(str), _enum_to_value(delim), _enum_to_value(count) - ) + return _invoke_function_over_columns("degrees", col) @_try_remote_functions -def levenshtein( - left: "ColumnOrName", right: "ColumnOrName", threshold: Optional[int] = None -) -> Column: - """Computes the Levenshtein distance of the two given strings. +def radians(col: "ColumnOrName") -> Column: + """ + Converts an angle measured in degrees to an approximately equivalent angle + measured in radians. - .. versionadded:: 1.5.0 + .. versionadded:: 2.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - first column value. - A column that evaluates to a string. - right : :class:`~pyspark.sql.Column` or column name - second column value. - A column that evaluates to a string. - threshold : int, optional - if set when the levenshtein distance of the two given strings - less than or equal to a given threshold then return result distance, or -1. - A column that evaluates to an integer. - - .. versionadded:: 3.5.0 + col : :class:`~pyspark.sql.Column` or column name + angle in degrees. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - Levenshtein distance as integer value. - Returns a column that evaluates to an integer. + angle in radians, as if computed by `java.lang.Math.toRadians()` + Returns a column that evaluates to a double. + + See Also + -------- + :meth:`pyspark.sql.functions.degrees` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r']) - >>> df.select('*', sf.levenshtein('l', 'r')).show() - +------+-------+-----------------+ - | l| r|levenshtein(l, r)| - +------+-------+-----------------+ - |kitten|sitting| 3| - +------+-------+-----------------+ - - >>> df.select('*', sf.levenshtein(df.l, df.r, 2)).show() - +------+-------+--------------------+ - | l| r|levenshtein(l, r, 2)| - +------+-------+--------------------+ - |kitten|sitting| -1| - +------+-------+--------------------+ + >>> spark.sql( + ... "SELECT * FROM VALUES (180), (90), (45), (0) AS TAB(value)" + ... ).select("*", sf.radians("value")).show() + +-----+------------------+ + |value| RADIANS(value)| + +-----+------------------+ + | 180| 3.141592653589...| + | 90|1.5707963267948...| + | 45|0.7853981633974...| + | 0| 0.0| + +-----+------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if threshold is None: - return _invoke_function_over_columns("levenshtein", left, right) - else: - return _invoke_function( - "levenshtein", _to_java_column(left), _to_java_column(right), _enum_to_value(threshold) - ) + return _invoke_function_over_columns("radians", col) @_try_remote_functions -def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Computes the Jaro-Winkler similarity between the two given strings. +def atan2(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: + """ + Compute the angle in radians between the positive x-axis of a plane + and the point given by the coordinates - The result is a double between 0.0 (no similarity) and 1.0 (identical strings). + .. versionadded:: 1.4.0 - .. versionadded:: 4.3.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - first column value. - A column that evaluates to a string. - right : :class:`~pyspark.sql.Column` or column name - second column value. - A column that evaluates to a string. + col1 : :class:`~pyspark.sql.Column`, column name or float + coordinate on y-axis. + A column that evaluates to a double. + col2 : :class:`~pyspark.sql.Column`, column name or float + coordinate on x-axis. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - Jaro-Winkler similarity as a double value. + the `theta` component of the point + (`r`, `theta`) + in polar coordinates that corresponds to the point + (`x`, `y`) in Cartesian coordinates, + as if computed by `java.lang.Math.atan2()` Returns a column that evaluates to a double. + See Also + -------- + :meth:`pyspark.sql.functions.atan` + :meth:`pyspark.sql.functions.hypot` + Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('MARTHA', 'MARHTA')], ['l', 'r']) - >>> df.select(sf.jaro_winkler_similarity('l', 'r')).show() - +-----------------------------+ - |jaro_winkler_similarity(l, r)| - +-----------------------------+ - | 0.9611111111111111| - +-----------------------------+ + >>> spark.range(1).select(sf.atan2(sf.lit(1), sf.lit(2))).show() + +------------------+ + | ATAN2(1, 2)| + +------------------+ + |0.4636476090008...| + +------------------+ """ - return _invoke_function_over_columns("jaro_winkler_similarity", left, right) + return _invoke_binary_math_function("atan2", col1, col2) @_try_remote_functions -def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: +def hypot(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: """ - Locate the position of the first occurrence of substr in a string column, after position pos. + Computes ``sqrt(a^2 + b^2)`` without intermediate overflow or underflow. - .. versionadded:: 1.5.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - substr : literal string - a string. - A column that evaluates to a string. - str : :class:`~pyspark.sql.Column` or column name - a Column of :class:`pyspark.sql.types.StringType`. - A column that evaluates to a string. - pos : int, optional - start position (zero based). - A column that evaluates to an integer. + col1 : :class:`~pyspark.sql.Column`, column name or float + a leg. + A column that evaluates to a double. + col2 : :class:`~pyspark.sql.Column`, column name or float + b leg. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - position of the substring. - Returns a column that evaluates to an integer. - - Notes - ----- - The position is not zero based, but 1 based index. Returns 0 if substr - could not be found in str. - - See Also - -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.substr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.Column.substr` + length of the hypotenuse. + Returns a column that evaluates to a double. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',)], ['s',]) - >>> df.select('*', sf.locate('b', 's', 1)).show() - +----+---------------+ - | s|locate(b, s, 1)| - +----+---------------+ - |abcd| 2| - +----+---------------+ - - >>> df.select('*', sf.locate('b', df.s, 3)).show() - +----+---------------+ - | s|locate(b, s, 3)| - +----+---------------+ - |abcd| 0| - +----+---------------+ + >>> spark.range(1).select(sf.hypot(sf.lit(1), sf.lit(2))).show() + +----------------+ + | HYPOT(1, 2)| + +----------------+ + |2.23606797749...| + +----------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "locate", _enum_to_value(substr), _to_java_column(str), _enum_to_value(pos) - ) + return _invoke_binary_math_function("hypot", col1, col2) @_try_remote_functions -def lpad( - col: "ColumnOrName", - len: Union[Column, int], - pad: Union[Column, str], -) -> Column: +def pow(col1: Union["ColumnOrName", float], col2: Union["ColumnOrName", float]) -> Column: """ - Left-pad the string column to width `len` with `pad`. + Returns the value of the first argument raised to the power of the second argument. - .. versionadded:: 1.5.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string or binary. - len : :class:`~pyspark.sql.Column` or int - length of the final string. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. - - pad : :class:`~pyspark.sql.Column` or literal string - chars to prepend. - A column that evaluates to a string or binary. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. + col1 : :class:`~pyspark.sql.Column`, column name or float + the base number. + A column that evaluates to a double. + col2 : :class:`~pyspark.sql.Column`, column name or float + the exponent number. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - left padded result. - Returns a column of the same type as the input. - - See Also - -------- - :meth:`pyspark.sql.functions.rpad` + the base rased to the power the argument. + Returns a column that evaluates to a double. Examples -------- - Example 1: Pad with a literal string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.lpad(df.s, 6, '#')).show() - +----+-------------+ - | s|lpad(s, 6, #)| - +----+-------------+ - |abcd| ##abcd| - | xyz| ###xyz| - | 12| ####12| - +----+-------------+ + >>> spark.range(5).select("*", sf.pow("id", 2)).show() + +---+------------+ + | id|POWER(id, 2)| + +---+------------+ + | 0| 0.0| + | 1| 1.0| + | 2| 4.0| + | 3| 9.0| + | 4| 16.0| + +---+------------+ + """ + return _invoke_binary_math_function("pow", col1, col2) - Example 2: Pad with a bytes column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.lpad(df.s, 6, sf.lit(b"\x75\x76"))).show() - +----+-------------------+ - | s|lpad(s, 6, X'7576')| - +----+-------------------+ - |abcd| uvabcd| - | xyz| uvuxyz| - | 12| uvuv12| - +----+-------------------+ - """ - return _invoke_function_over_columns("lpad", col, lit(len), lit(pad)) +power = pow @_try_remote_functions -def rpad( - col: "ColumnOrName", - len: Union[Column, int], - pad: Union[Column, str], -) -> Column: +def pmod(dividend: Union["ColumnOrName", float], divisor: Union["ColumnOrName", float]) -> Column: """ - Right-pad the string column to width `len` with `pad`. - - .. versionadded:: 1.5.0 + Returns the positive value of dividend mod divisor. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target column to work on. - A column that evaluates to a string or binary. - len : :class:`~pyspark.sql.Column` or int - length of the final string. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. - - pad : :class:`~pyspark.sql.Column` or literal string - chars to prepend. - A column that evaluates to a string or binary. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. + dividend : :class:`~pyspark.sql.Column`, column name or float + the column that contains dividend, or the specified dividend value. + A column that evaluates to a numeric. + divisor : :class:`~pyspark.sql.Column`, column name or float + the column that contains divisor, or the specified divisor value. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - right padded result. + positive value of dividend mod divisor. Returns a column of the same type as the input. - See Also - -------- - :meth:`pyspark.sql.functions.lpad` + Notes + ----- + Supports Spark Connect. Examples -------- - Example 1: Pad with a literal string - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.rpad(df.s, 6, '#')).show() - +----+-------------+ - | s|rpad(s, 6, #)| - +----+-------------+ - |abcd| abcd##| - | xyz| xyz###| - | 12| 12####| - +----+-------------+ - - Example 2: Pad with a bytes column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) - >>> df.select("*", sf.rpad(df.s, 6, sf.lit(b"\x75\x76"))).show() - +----+-------------------+ - | s|rpad(s, 6, X'7576')| - +----+-------------------+ - |abcd| abcduv| - | xyz| xyzuvu| - | 12| 12uvuv| - +----+-------------------+ + >>> df = spark.createDataFrame([ + ... (1.0, float('nan')), (float('nan'), 2.0), (10.0, 3.0), + ... (float('nan'), float('nan')), (-3.0, 4.0), (-10.0, 3.0), + ... (-5.0, -6.0), (7.0, -8.0), (1.0, 2.0)], + ... ("a", "b")) + >>> df.select("*", sf.pmod("a", "b")).show() + +-----+----+----------+ + | a| b|pmod(a, b)| + +-----+----+----------+ + | 1.0| NaN| NaN| + | NaN| 2.0| NaN| + | 10.0| 3.0| 1.0| + | NaN| NaN| NaN| + | -3.0| 4.0| 1.0| + |-10.0| 3.0| 2.0| + | -5.0|-6.0| -5.0| + | 7.0|-8.0| 7.0| + | 1.0| 2.0| 1.0| + +-----+----+----------+ """ - return _invoke_function_over_columns("rpad", col, lit(len), lit(pad)) + return _invoke_binary_math_function("pmod", dividend, divisor) @_try_remote_functions -def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: +def width_bucket( + v: "ColumnOrName", + min: "ColumnOrName", + max: "ColumnOrName", + numBucket: Union["ColumnOrName", int], +) -> Column: """ - Repeats a string column n times, and returns it as a new string column. - - .. versionadded:: 1.5.0 + Returns the bucket number into which the value of this expression would fall + after being evaluated. Note that input arguments must follow conditions listed below; + otherwise, the method will return null. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - n : :class:`~pyspark.sql.Column` or column name or int - number of times to repeat value. - A column that evaluates to an integer. - - .. versionchanged:: 4.0.0 - `n` now accepts column and column name. + v : :class:`~pyspark.sql.Column` or column name + value to compute a bucket number in the histogram. + A column that evaluates to a double or interval. + min : :class:`~pyspark.sql.Column` or column name + minimum value of the histogram. + A column that evaluates to a double or interval. + max : :class:`~pyspark.sql.Column` or column name + maximum value of the histogram. + A column that evaluates to a double or interval. + numBucket : :class:`~pyspark.sql.Column`, column name or int + the number of buckets. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` - string with repeated values. - Returns a column that evaluates to a string. + the bucket number into which the value would fall after being evaluated + Returns a column that evaluates to a long. Examples -------- - Example 1: Repeat with a constant number of times - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ab',)], ['s',]) - >>> df.select("*", sf.repeat("s", 3)).show() - +---+------------+ - | s|repeat(s, 3)| - +---+------------+ - | ab| ababab| - +---+------------+ - - >>> df.select("*", sf.repeat(df.s, sf.lit(4))).show() - +---+------------+ - | s|repeat(s, 4)| - +---+------------+ - | ab| abababab| - +---+------------+ - - Example 2: Repeat with a column containing different number of times - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ab', 5,), ('abc', 6,)], ['s', 't']) - >>> df.select("*", sf.repeat("s", "t")).show() - +---+---+------------------+ - | s| t| repeat(s, t)| - +---+---+------------------+ - | ab| 5| ababababab| - |abc| 6|abcabcabcabcabcabc| - +---+---+------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (5.3, 0.2, 10.6, 5), + ... (-2.1, 1.3, 3.4, 3), + ... (8.1, 0.0, 5.7, 4), + ... (-0.9, 5.2, 0.5, 2)], + ... ['v', 'min', 'max', 'n']) + >>> df.select("*", sf.width_bucket('v', 'min', 'max', 'n')).show() + +----+---+----+---+----------------------------+ + | v|min| max| n|width_bucket(v, min, max, n)| + +----+---+----+---+----------------------------+ + | 5.3|0.2|10.6| 5| 3| + |-2.1|1.3| 3.4| 3| 0| + | 8.1|0.0| 5.7| 4| 5| + |-0.9|5.2| 0.5| 2| 3| + +----+---+----+---+----------------------------+ """ - n = _enum_to_value(n) - n = lit(n) if isinstance(n, int) else n - return _invoke_function_over_columns("repeat", col, n) + numBucket = _enum_to_value(numBucket) + numBucket = lit(numBucket) if isinstance(numBucket, int) else numBucket + return _invoke_function_over_columns("width_bucket", v, min, max, numBucket) @_try_remote_functions -def split( - str: "ColumnOrName", - pattern: Union[Column, str], - limit: Union["ColumnOrName", int] = -1, -) -> Column: +def row_number() -> Column: """ - Splits str around matches of the given pattern. + Window function: returns a sequential number starting at 1 within a window partition. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - a string expression to split. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or literal string - a string representing a regular expression. The regex string should be - a Java regular expression. - A column that evaluates to a string. - - .. versionchanged:: 4.0.0 - `pattern` now accepts column. Does not accept column name since string type remain - accepted as a regular expression representation, for backwards compatibility. - In addition to int, `limit` now accepts column and column name. - - limit : :class:`~pyspark.sql.Column` or column name or int - an integer which controls the number of times `pattern` is applied. - A column that evaluates to an integer. - - * ``limit > 0``: The resulting array's length will not be more than `limit`, and the - resulting array's last entry will contain all input beyond the last - matched pattern. - * ``limit <= 0``: `pattern` will be applied as many times as possible, and the resulting - array can be of any size. - - .. versionchanged:: 3.0 - `split` now takes an optional `limit` field. If not provided, default limit value is -1. - Returns ------- :class:`~pyspark.sql.Column` - array of separated strings. - Returns a column that evaluates to an array. + the column for calculating row numbers. See Also -------- - :meth:`pyspark.sql.functions.sentences` - :meth:`pyspark.sql.functions.split_part` + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` Examples -------- - Example 1: Repeat with a constant pattern + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.range(3) + >>> w = Window.orderBy(df.id.desc()) + >>> df.withColumn("desc_order", sf.row_number().over(w)).show() + +---+----------+ + | id|desc_order| + +---+----------+ + | 2| 1| + | 1| 2| + | 0| 3| + +---+----------+ + """ + return _invoke_function("row_number") - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',]) - >>> df.select('*', sf.split(df.s, '[ABC]')).show() - +--------------+-------------------+ - | s|split(s, [ABC], -1)| - +--------------+-------------------+ - |oneAtwoBthreeC|[one, two, three, ]| - +--------------+-------------------+ - >>> df.select('*', sf.split(df.s, '[ABC]', 2)).show() - +--------------+------------------+ - | s|split(s, [ABC], 2)| - +--------------+------------------+ - |oneAtwoBthreeC| [one, twoBthreeC]| - +--------------+------------------+ +@_try_remote_functions +def dense_rank() -> Column: + """ + Window function: returns the rank of rows within a window partition, without any gaps. - >>> df.select('*', sf.split('s', '[ABC]', -2)).show() - +--------------+-------------------+ - | s|split(s, [ABC], -2)| - +--------------+-------------------+ - |oneAtwoBthreeC|[one, two, three, ]| - +--------------+-------------------+ + The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking + sequence when there are ties. That is, if you were ranking a competition using dense_rank + and had three people tie for second place, you would say that all three were in second + place and that the next person came in third. Rank would give me sequential numbers, making + the person that came in third place (after the ties) would register as coming in fifth. - Example 2: Repeat with a column containing different patterns and limits + This is equivalent to the DENSE_RANK function in SQL. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ('oneAtwoBthreeC', '[ABC]', 2), - ... ('1A2B3C', '[1-9]+', 1), - ... ('aa2bb3cc4', '[1-9]+', -1)], ['s', 'p', 'l']) - >>> df.select('*', sf.split(df.s, df.p)).show() - +--------------+------+---+-------------------+ - | s| p| l| split(s, p, -1)| - +--------------+------+---+-------------------+ - |oneAtwoBthreeC| [ABC]| 2|[one, two, three, ]| - | 1A2B3C|[1-9]+| 1| [, A, B, C]| - | aa2bb3cc4|[1-9]+| -1| [aa, bb, cc, ]| - +--------------+------+---+-------------------+ + .. versionadded:: 1.6.0 - >>> df.select(sf.split('s', df.p, 'l')).show() - +-----------------+ - | split(s, p, l)| - +-----------------+ - |[one, twoBthreeC]| - | [1A2B3C]| - | [aa, bb, cc, ]| - +-----------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for calculating ranks. + + See Also + -------- + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("drank", sf.dense_rank().over(w)).show() + +-----+-----+ + |value|drank| + +-----+-----+ + | 1| 1| + | 1| 1| + | 2| 2| + | 3| 3| + | 3| 3| + | 4| 4| + +-----+-----+ """ - limit = _enum_to_value(limit) - limit = lit(limit) if isinstance(limit, int) else limit - return _invoke_function_over_columns("split", str, lit(pattern), limit) + return _invoke_function("dense_rank") @_try_remote_functions -def randstr(length: Union[Column, int], seed: Optional[Union[Column, int]] = None) -> Column: - """Returns a string of the specified length whose characters are chosen uniformly at random from - the following pool of characters: 0-9, a-z, A-Z. The random seed is optional. The string length - must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). +def rank() -> Column: + """ + Window function: returns the rank of rows within a window partition. - .. versionadded:: 4.0.0 + The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking + sequence when there are ties. That is, if you were ranking a competition using dense_rank + and had three people tie for second place, you would say that all three were in second + place and that the next person came in third. Rank would give me sequential numbers, making + the person that came in third place (after the ties) would register as coming in fifth. - Parameters - ---------- - length : :class:`~pyspark.sql.Column` or int - Number of characters in the string to generate. - A column that evaluates to an integer. Must be a constant. - seed : :class:`~pyspark.sql.Column` or int - Optional random number seed to use. - A column that evaluates to an integer or long. Must be a constant. + This is equivalent to the RANK function in SQL. + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Returns ------- :class:`~pyspark.sql.Column` - The generated random string with the specified length. - Returns a column that evaluates to a string. + the column for calculating ranks. See Also -------- - :meth:`pyspark.sql.functions.rand` - :meth:`pyspark.sql.functions.randn` + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.row_number` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(0, 10, 1, 1).select(sf.randstr(16, 3)).show() - +----------------+ - | randstr(16, 3)| - +----------------+ - |nurJIpH4cmmMnsCG| - |fl9YtT5m01trZtIt| - |PD19rAgscTHS7qQZ| - |2CuAICF5UJOruVv4| - |kNZEs8nDpJEoz3Rl| - |OXiU0KN5eaXfjXFs| - |qfnTM1BZAHtN0gBV| - |1p8XiSKwg33KnRPK| - |od5y5MucayQq1bKK| - |tklYPmKmc5sIppWM| - +----------------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("drank", sf.rank().over(w)).show() + +-----+-----+ + |value|drank| + +-----+-----+ + | 1| 1| + | 1| 1| + | 2| 3| + | 3| 4| + | 3| 4| + | 4| 6| + +-----+-----+ """ - length = _enum_to_value(length) - length = lit(length) - if seed is None: - return _invoke_function_over_columns("randstr", length) - else: - seed = _enum_to_value(seed) - seed = lit(seed) - return _invoke_function_over_columns("randstr", length, seed) + return _invoke_function("rank") @_try_remote_functions -def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns a count of the number of times that the Java regex pattern `regexp` is matched - in the string `str`. +def counter_diff(value: "ColumnOrName", startTime: Optional["ColumnOrName"] = None) -> Column: + """ + Window function: computes the differences between consecutive cumulative counter values in a + time series, thereby converting the counter from the cumulative to the delta format. - .. versionadded:: 3.5.0 + Gracefully handles counter resets by returning NULL. Counter resets are detected when the + counter value decreases, or when the start time advances between rows. + + Use the PARTITION BY clause of the window to separate independent counters. This is done by + specifying all columns which uniquely identify a time series. These are typically the counter + name and any attributes tied to the counter. + + Use the ORDER BY clause of the window to order the observations by the associated timestamp + in ascending order. + + .. versionadded:: 4.3.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. + value : :class:`~pyspark.sql.Column` or column name + A cumulative counter. Must be a numeric data type. Must be non-negative. + startTime : :class:`~pyspark.sql.Column` or column name, optional + An optional timestamp parameter which indicates when the counter was last set to zero. + Used to signal counter resets. Returns ------- :class:`~pyspark.sql.Column` - the number of times that a Java regex pattern is matched in the string. - Returns a column that evaluates to an integer. + The difference between the current and previous counter value within the window partition. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) - >>> df.select('*', sf.regexp_count('str', sf.lit(r'\d+'))).show() - +---------+------+----------------------+ - | str|regexp|regexp_count(str, \d+)| - +---------+------+----------------------+ - |1a 2b 14m| \d+| 3| - +---------+------+----------------------+ - - >>> df.select('*', sf.regexp_count('str', sf.lit(r'mmm'))).show() - +---------+------+----------------------+ - | str|regexp|regexp_count(str, mmm)| - +---------+------+----------------------+ - |1a 2b 14m| \d+| 0| - +---------+------+----------------------+ - - >>> df.select('*', sf.regexp_count("str", sf.col("regexp"))).show() - +---------+------+-------------------------+ - | str|regexp|regexp_count(str, regexp)| - +---------+------+-------------------------+ - |1a 2b 14m| \d+| 3| - +---------+------+-------------------------+ + >>> from pyspark.sql import Window + >>> from datetime import datetime + >>> df = spark.createDataFrame( + ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100), + ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200), + ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400), + ... ('http_requests', datetime(2026, 1, 1, 0, 3), 50), + ... ('http_requests', datetime(2026, 1, 1, 0, 4), 100)], + ... "m STRING, t TIMESTAMP_NTZ, c INT") + >>> w = Window.partitionBy("m").orderBy("t") + >>> df.select("m", "t", "c", sf.counter_diff("c").over(w).alias("diff")).show() + +-------------+-------------------+---+----+ + | m| t| c|diff| + +-------------+-------------------+---+----+ + |http_requests|2026-01-01 00:00:00|100|NULL| + |http_requests|2026-01-01 00:01:00|200| 100| + |http_requests|2026-01-01 00:02:00|400| 200| + |http_requests|2026-01-01 00:03:00| 50|NULL| + |http_requests|2026-01-01 00:04:00|100| 50| + +-------------+-------------------+---+----+ - >>> df.select('*', sf.regexp_count(sf.col('str'), "regexp")).show() - +---------+------+-------------------------+ - | str|regexp|regexp_count(str, regexp)| - +---------+------+-------------------------+ - |1a 2b 14m| \d+| 3| - +---------+------+-------------------------+ + >>> df2 = spark.createDataFrame( + ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100, datetime(2026, 1, 1, 0, 0)), + ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200, datetime(2026, 1, 1, 0, 0)), + ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400, datetime(2026, 1, 1, 0, 0)), + ... ('http_requests', datetime(2026, 1, 1, 0, 3), 500, datetime(2026, 1, 1, 0, 2, 15)), + ... ('http_requests', datetime(2026, 1, 1, 0, 4), 600, datetime(2026, 1, 1, 0, 2, 15))], + ... "m STRING, t TIMESTAMP_NTZ, c INT, s TIMESTAMP_NTZ") + >>> df2.select("m", "t", "s", "c", sf.counter_diff("c", "s").over(w).alias("diff")).show() + +-------------+-------------------+-------------------+---+----+ + | m| t| s| c|diff| + +-------------+-------------------+-------------------+---+----+ + |http_requests|2026-01-01 00:00:00|2026-01-01 00:00:00|100|NULL| + |http_requests|2026-01-01 00:01:00|2026-01-01 00:00:00|200| 100| + |http_requests|2026-01-01 00:02:00|2026-01-01 00:00:00|400| 200| + |http_requests|2026-01-01 00:03:00|2026-01-01 00:02:15|500|NULL| + |http_requests|2026-01-01 00:04:00|2026-01-01 00:02:15|600| 100| + +-------------+-------------------+-------------------+---+----+ """ - return _invoke_function_over_columns("regexp_count", str, regexp) + if startTime is None: + return _invoke_function_over_columns("counter_diff", value) + return _invoke_function_over_columns("counter_diff", value, startTime) @_try_remote_functions -def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: - r"""Extract a specific group matched by the Java regex `regexp`, from the specified string column. - If the regex did not match, or the specified group did not match, an empty string is returned. +def cume_dist() -> Column: + """ + Window function: returns the cumulative distribution of values within a window partition, + i.e. the fraction of rows that are below the current row. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - pattern : str - regex pattern to apply. - A column that evaluates to a string. - idx : int - matched group id. - A column that evaluates to an integer. - Returns ------- :class:`~pyspark.sql.Column` - matched value specified by `idx` group id. - Returns a column that evaluates to a string. + the column for calculating cumulative distribution. See Also -------- - :meth:`pyspark.sql.functions.regexp_extract_all` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('100-200',)], ['str']) - >>> df.select('*', sf.regexp_extract('str', r'(\d+)-(\d+)', 1)).show() - +-------+-----------------------------------+ - | str|regexp_extract(str, (\d+)-(\d+), 1)| - +-------+-----------------------------------+ - |100-200| 100| - +-------+-----------------------------------+ - - >>> df = spark.createDataFrame([('foo',)], ['str']) - >>> df.select('*', sf.regexp_extract('str', r'(\d+)', 1)).show() - +---+-----------------------------+ - |str|regexp_extract(str, (\d+), 1)| - +---+-----------------------------+ - |foo| | - +---+-----------------------------+ - - >>> df = spark.createDataFrame([('aaaac',)], ['str']) - >>> df.select('*', sf.regexp_extract(sf.col('str'), '(a+)(b)?(c)', 2)).show() - +-----+-----------------------------------+ - | str|regexp_extract(str, (a+)(b)?(c), 2)| - +-----+-----------------------------------+ - |aaaac| | - +-----+-----------------------------------+ + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("cd", sf.cume_dist().over(w)).show() + +-----+---+ + |value| cd| + +-----+---+ + | 1|0.2| + | 2|0.4| + | 3|0.8| + | 3|0.8| + | 4|1.0| + +-----+---+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "regexp_extract", _to_java_column(str), _enum_to_value(pattern), _enum_to_value(idx) - ) + return _invoke_function("cume_dist") @_try_remote_functions -def regexp_extract_all( - str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None -) -> Column: - r"""Extract all strings in the `str` that match the Java regex `regexp` - and corresponding to the regex group index. +def percent_rank() -> Column: + """ + Window function: returns the relative rank (i.e. percentile) of rows within a window partition. - .. versionadded:: 3.5.0 + .. versionadded:: 1.6.0 - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. - idx : :class:`~pyspark.sql.Column` or int, optional - matched group id. - A column that evaluates to an integer. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Returns ------- :class:`~pyspark.sql.Column` - all strings in the `str` that match a Java regex and corresponding to the regex group index. - Returns a column that evaluates to an array. + the column for calculating relative rank. See Also -------- - :meth:`pyspark.sql.functions.regexp_extract` + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.ntile` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("100-200, 300-400", r"(\d+)-(\d+)")], ["str", "regexp"]) - >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'))).show() - +----------------+-----------+---------------------------------------+ - | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| - +----------------+-----------+---------------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+---------------------------------------+ + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") + >>> w = Window.orderBy("value") + >>> df.withColumn("pr", sf.percent_rank().over(w)).show() + +-----+---+ + |value| pr| + +-----+---+ + | 1|0.0| + | 1|0.0| + | 2|0.4| + | 3|0.6| + | 3|0.6| + | 4|1.0| + +-----+---+ + """ + return _invoke_function("percent_rank") - >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), sf.lit(1))).show() - +----------------+-----------+---------------------------------------+ - | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| - +----------------+-----------+---------------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+---------------------------------------+ - >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), 2)).show() - +----------------+-----------+---------------------------------------+ - | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 2)| - +----------------+-----------+---------------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [200, 400]| - +----------------+-----------+---------------------------------------+ +@_try_remote_functions +def approxCountDistinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: + """ + This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates + the approximate distinct count of elements in a specified column or a group of columns. - >>> df.select('*', sf.regexp_extract_all('str', sf.col("regexp"))).show() - +----------------+-----------+----------------------------------+ - | str| regexp|regexp_extract_all(str, regexp, 1)| - +----------------+-----------+----------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+----------------------------------+ + .. versionadded:: 1.3.0 - >>> df.select('*', sf.regexp_extract_all(sf.col('str'), "regexp")).show() - +----------------+-----------+----------------------------------+ - | str| regexp|regexp_extract_all(str, regexp, 1)| - +----------------+-----------+----------------------------------+ - |100-200, 300-400|(\d+)-(\d+)| [100, 300]| - +----------------+-----------+----------------------------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 2.1.0 + Use :func:`approx_count_distinct` instead. """ - if idx is None: - return _invoke_function_over_columns("regexp_extract_all", str, regexp) - else: - return _invoke_function_over_columns("regexp_extract_all", str, regexp, lit(idx)) + warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", FutureWarning) + return approx_count_distinct(col, rsd) @_try_remote_functions -def regexp_replace( - string: "ColumnOrName", - pattern: Union[str, Column], - replacement: Union[str, Column], - position: Optional[Union[int, Column]] = None, -) -> Column: - r"""Replace all substrings of the specified string value that match regexp with replacement. +def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: + """ + This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates + the approximate distinct count of elements in a specified column or a group of columns. - .. versionadded:: 1.5.0 + .. versionadded:: 2.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.3.0 - Supports the `position` parameter. Parameters ---------- - string : :class:`~pyspark.sql.Column` or str - column name or column containing the string value. - A column that evaluates to a string. - pattern : :class:`~pyspark.sql.Column` or str - column object or str containing the regexp pattern. - A column that evaluates to a string. - replacement : :class:`~pyspark.sql.Column` or str - column object or str containing the replacement. - A column that evaluates to a string. - position : :class:`~pyspark.sql.Column` or int, optional - position to start replacement. The first position is 1. - A column that evaluates to an integer. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + The label of the column to count distinct values in. + rsd : float, optional + The maximum allowed relative standard deviation (default = 0.05). + If rsd < 0.01, it would be more efficient to use :func:`count_distinct`. Returns ------- :class:`~pyspark.sql.Column` - string with all substrings replaced. - Returns a column that evaluates to a string. + A new Column object representing the approximate unique count. + + See Also + -------- + :meth:`pyspark.sql.functions.count_distinct` Examples -------- + Example 1: Counting distinct values in a single column DataFrame representing integers + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("100-200", r"(\d+)", "--")], - ... ["str", "pattern", "replacement"] - ... ) + >>> df = spark.createDataFrame([1,2,2,3], "int") + >>> df.agg(sf.approx_count_distinct("value")).show() + +----------------------------+ + |approx_count_distinct(value)| + +----------------------------+ + | 3| + +----------------------------+ - Example 1: Replaces all the substrings in the `str` column name that - match the regex pattern `(\d+)` (one or more digits) with the replacement - string "--". + Example 2: Counting distinct values in a single column DataFrame representing strings - >>> df.select('*', sf.regexp_replace('str', r'(\d+)', '--')).show() - +-------+-------+-----------+---------------------------------+ - | str|pattern|replacement|regexp_replace(str, (\d+), --, 1)| - +-------+-------+-----------+---------------------------------+ - |100-200| (\d+)| --| -----| - +-------+-------+-----------+---------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("apple",), ("orange",), ("apple",), ("banana",)], ['fruit']) + >>> df.agg(sf.approx_count_distinct("fruit")).show() + +----------------------------+ + |approx_count_distinct(fruit)| + +----------------------------+ + | 3| + +----------------------------+ - Example 2: Replaces all the substrings in the `str` Column that match - the regex pattern in the `pattern` Column with the string in the `replacement` - column. + Example 3: Counting distinct values in a DataFrame with multiple columns - >>> df.select('*', \ - ... sf.regexp_replace(sf.col("str"), sf.col("pattern"), sf.col("replacement")) \ - ... ).show() - +-------+-------+-----------+--------------------------------------------+ - | str|pattern|replacement|regexp_replace(str, pattern, replacement, 1)| - +-------+-------+-----------+--------------------------------------------+ - |100-200| (\d+)| --| -----| - +-------+-------+-----------+--------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("Alice", 1), ("Alice", 2), ("Bob", 3), ("Bob", 3)], ["name", "value"]) + >>> df = df.withColumn("combined", sf.struct("name", "value")) + >>> df.agg(sf.approx_count_distinct(df.combined)).show() + +-------------------------------+ + |approx_count_distinct(combined)| + +-------------------------------+ + | 3| + +-------------------------------+ - Example 3: Replaces substrings starting from the specified position. - For the input string "100-200", position 5 starts replacement after "100-". + Example 4: Counting distinct values with a specified relative standard deviation - >>> df.select(sf.regexp_replace("str", r"(\d+)", "--", 5).alias("d")).show() - +------+ - | d| - +------+ - |100---| - +------+ + >>> from pyspark.sql import functions as sf + >>> spark.range(100000).agg( + ... sf.approx_count_distinct("id").alias('with_default_rsd'), + ... sf.approx_count_distinct("id", 0.1).alias('with_rsd_0.1') + ... ).show() + +----------------+------------+ + |with_default_rsd|with_rsd_0.1| + +----------------+------------+ + | 95546| 102065| + +----------------+------------+ """ - if position is None: - return _invoke_function_over_columns( - "regexp_replace", string, lit(pattern), lit(replacement) - ) + from pyspark.sql.classic.column import _to_java_column + + if rsd is None: + return _invoke_function_over_columns("approx_count_distinct", col) else: - return _invoke_function_over_columns( - "regexp_replace", - string, - lit(pattern), - lit(replacement), - lit(position), - ) + return _invoke_function("approx_count_distinct", _to_java_column(col), _enum_to_value(rsd)) @_try_remote_functions -def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: - r"""Returns the first substring that matches the Java regex `regexp` within the string `str`. - If the regular expression is not found, the result is null. +def broadcast(df: "DataFrame") -> "DataFrame": + """ + Marks a DataFrame as small enough for use in broadcast joins. - .. versionadded:: 3.5.0 + .. versionadded:: 1.6.0 - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Returns ------- - :class:`~pyspark.sql.Column` - the first substring that matches a Java regex within the string `str`. - Returns a column that evaluates to a string. + :class:`~pyspark.sql.DataFrame` + DataFrame marked as ready for broadcast join. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) - - Example 1: Returns the first substring in the `str` column name that - matches the regex pattern `(\d+)` (one or more digits). - - >>> df.select('*', sf.regexp_substr('str', sf.lit(r'\d+'))).show() - +---------+------+-----------------------+ - | str|regexp|regexp_substr(str, \d+)| - +---------+------+-----------------------+ - |1a 2b 14m| \d+| 1| - +---------+------+-----------------------+ - - Example 2: Returns the first substring in the `str` column name that - matches the regex pattern `(mmm)` (three consecutive 'm' characters) - - >>> df.select('*', sf.regexp_substr('str', sf.lit(r'mmm'))).show() - +---------+------+-----------------------+ - | str|regexp|regexp_substr(str, mmm)| - +---------+------+-----------------------+ - |1a 2b 14m| \d+| NULL| - +---------+------+-----------------------+ - - Example 3: Returns the first substring in the `str` column name that - matches the regex pattern in `regexp` Column. - - >>> df.select('*', sf.regexp_substr("str", sf.col("regexp"))).show() - +---------+------+--------------------------+ - | str|regexp|regexp_substr(str, regexp)| - +---------+------+--------------------------+ - |1a 2b 14m| \d+| 1| - +---------+------+--------------------------+ + >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") + >>> df_small = spark.range(3) + >>> df_b = sf.broadcast(df_small) + >>> df.join(df_b, df.value == df_small.id).show() + +-----+---+ + |value| id| + +-----+---+ + | 1| 1| + | 2| 2| + +-----+---+ + """ + from py4j.java_gateway import JVMView - Example 4: Returns the first substring in the `str` Column that - matches the regex pattern in `regexp` column name. + from pyspark.sql.dataframe import DataFrame - >>> df.select('*', sf.regexp_substr(sf.col("str"), "regexp")).show() - +---------+------+--------------------------+ - | str|regexp|regexp_substr(str, regexp)| - +---------+------+--------------------------+ - |1a 2b 14m| \d+| 1| - +---------+------+--------------------------+ - """ - return _invoke_function_over_columns("regexp_substr", str, regexp) + sc = _get_active_spark_context() + return DataFrame(cast(JVMView, sc._jvm).functions.broadcast(df._jdf), df.sparkSession) @_try_remote_functions -def regexp_instr( - str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None -) -> Column: - r"""Returns the position of the first substring in the `str` that match the Java regex `regexp` - and corresponding to the regex group index. +def coalesce(*cols: "ColumnOrName") -> Column: + """Returns the first column that is not null. - .. versionadded:: 3.5.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. - regexp : :class:`~pyspark.sql.Column` or column name - regex pattern to apply. - A column that evaluates to a string. - idx : :class:`~pyspark.sql.Column` or int, optional - matched group id. - A column that evaluates to an integer. + cols : :class:`~pyspark.sql.Column` or column name + list of columns to work on. + Each a column of any type. Returns ------- :class:`~pyspark.sql.Column` - the position of the first substring in the `str` that match a Java regex and corresponding - to the regex group index. - Returns a column that evaluates to an integer. + value of the first column that is not null. + Returns a column of the same type as the input. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+(a|b|m)")], ["str", "regexp"]) - - Example 1: Returns the position of the first substring in the `str` column name that - match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'). - - >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'))).show() - +---------+----------+--------------------------------+ - | str| regexp|regexp_instr(str, \d+(a|b|m), 0)| - +---------+----------+--------------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+--------------------------------+ - - Example 2: Returns the position of the first substring in the `str` column name that - match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'), - - >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'), sf.lit(1))).show() - +---------+----------+--------------------------------+ - | str| regexp|regexp_instr(str, \d+(a|b|m), 1)| - +---------+----------+--------------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+--------------------------------+ - - Example 3: Returns the position of the first substring in the `str` column name that - match the regex pattern in `regexp` Column. - - >>> df.select('*', sf.regexp_instr('str', sf.col("regexp"))).show() - +---------+----------+----------------------------+ - | str| regexp|regexp_instr(str, regexp, 0)| - +---------+----------+----------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+----------------------------+ + >>> df = spark.createDataFrame([(None, None), (1, None), (None, 2)], ("a", "b")) + >>> df.show() + +----+----+ + | a| b| + +----+----+ + |NULL|NULL| + | 1|NULL| + |NULL| 2| + +----+----+ - Example 4: Returns the position of the first substring in the `str` Column that - match the regex pattern in `regexp` column name. + >>> df.select('*', sf.coalesce("a", df["b"])).show() + +----+----+--------------+ + | a| b|coalesce(a, b)| + +----+----+--------------+ + |NULL|NULL| NULL| + | 1|NULL| 1| + |NULL| 2| 2| + +----+----+--------------+ - >>> df.select('*', sf.regexp_instr(sf.col("str"), "regexp")).show() - +---------+----------+----------------------------+ - | str| regexp|regexp_instr(str, regexp, 0)| - +---------+----------+----------------------------+ - |1a 2b 14m|\d+(a|b|m)| 1| - +---------+----------+----------------------------+ + >>> df.select('*', sf.coalesce(df["a"], lit(0.0))).show() + +----+----+----------------+ + | a| b|coalesce(a, 0.0)| + +----+----+----------------+ + |NULL|NULL| 0.0| + | 1|NULL| 1.0| + |NULL| 2| 0.0| + +----+----+----------------+ """ - if idx is None: - return _invoke_function_over_columns("regexp_instr", str, regexp) - else: - return _invoke_function_over_columns("regexp_instr", str, regexp, lit(idx)) + return _invoke_function_over_seq_of_columns("coalesce", cols) @_try_remote_functions -def initcap(col: "ColumnOrName") -> Column: - """Translate the first letter of each word to upper case in the sentence. +def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for + ``col1`` and ``col2``. - .. versionadded:: 1.5.0 + .. versionadded:: 1.6.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + col1 : :class:`~pyspark.sql.Column` or column name + first column to calculate correlation. + A column that evaluates to a numeric. + col2 : :class:`~pyspark.sql.Column` or column name + second column to calculate correlation. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - string with all first letters are uppercase in each word. - Returns a column that evaluates to a string. + Pearson Correlation Coefficient of these two column values. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ab cd',)], ['a']) - >>> df.select("*", sf.initcap("a")).show() - +-----+----------+ - | a|initcap(a)| - +-----+----------+ - |ab cd| Ab Cd| - +-----+----------+ + >>> from pyspark.sql import functions as sf + >>> a = range(20) + >>> b = [2 * x for x in range(20)] + >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) + >>> df.agg(sf.corr("a", df.b)).show() + +----------+ + |corr(a, b)| + +----------+ + | 1.0| + +----------+ """ - return _invoke_function_over_columns("initcap", col) + return _invoke_function_over_columns("corr", col1, col2) @_try_remote_functions -def soundex(col: "ColumnOrName") -> Column: - """ - Returns the SoundEx encoding for a string +def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and + ``col2``. - .. versionadded:: 1.5.0 + .. versionadded:: 2.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string. + col1 : :class:`~pyspark.sql.Column` or column name + first column to calculate covariance. + A column that evaluates to a numeric. + col2 : :class:`~pyspark.sql.Column` or column name + second column to calculate covariance. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - SoundEx encoded string. - Returns a column that evaluates to a string. + covariance of these two column values. + + See Also + -------- + :meth:`pyspark.sql.functions.covar_samp` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Peters",),("Uhrbach",)], ["s"]) - >>> df.select("*", sf.soundex("s")).show() - +-------+----------+ - | s|soundex(s)| - +-------+----------+ - | Peters| P362| - |Uhrbach| U612| - +-------+----------+ + >>> from pyspark.sql import functions as sf + >>> a = [1] * 10 + >>> b = [1] * 10 + >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) + >>> df.agg(sf.covar_pop("a", df.b)).show() + +---------------+ + |covar_pop(a, b)| + +---------------+ + | 0.0| + +---------------+ """ - return _invoke_function_over_columns("soundex", col) + return _invoke_function_over_columns("covar_pop", col1, col2) @_try_remote_functions -def length(col: "ColumnOrName") -> Column: - """Computes the character length of string data or number of bytes of binary data. - The length of character data includes the trailing spaces. The length of binary data - includes binary zeros. +def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and + ``col2``. - .. versionadded:: 1.5.0 + .. versionadded:: 2.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a string or binary. + col1 : :class:`~pyspark.sql.Column` or column name + first column to calculate covariance. + A column that evaluates to a numeric. + col2 : :class:`~pyspark.sql.Column` or column name + second column to calculate covariance. + A column that evaluates to a numeric. Returns ------- :class:`~pyspark.sql.Column` - length of the value. - Returns a column that evaluates to an integer. + sample covariance of these two column values. See Also -------- - :meth:`pyspark.sql.functions.char_length` - :meth:`pyspark.sql.functions.character_length` + :meth:`pyspark.sql.functions.covar_pop` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.createDataFrame([('ABC ',)], ['a']).select('*', sf.length('a')).show() - +----+---------+ - | a|length(a)| - +----+---------+ - |ABC | 4| - +----+---------+ + >>> a = [1] * 10 + >>> b = [1] * 10 + >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) + >>> df.agg(sf.covar_samp("a", df.b)).show() + +----------------+ + |covar_samp(a, b)| + +----------------+ + | 0.0| + +----------------+ """ - return _invoke_function_over_columns("length", col) + return _invoke_function_over_columns("covar_samp", col1, col2) @_try_remote_functions -def octet_length(col: "ColumnOrName") -> Column: - """ - Calculates the byte length for the specified string column. +def countDistinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: + """Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``. - .. versionadded:: 3.3.0 + An alias of :func:`count_distinct`, and it is encouraged to use :func:`count_distinct` + directly. + + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - Source column or strings. - A column that evaluates to a string or binary. - - Returns - ------- - :class:`~pyspark.sql.Column` - Byte length of the col - Returns a column that evaluates to an integer. - Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) - >>> df.select('*', sf.octet_length('cat')).show() - +---+-----------------+ - |cat|octet_length(cat)| - +---+-----------------+ - |cat| 3| - | 🐈| 4| - +---+-----------------+ + >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) + >>> df.select(sf.count_distinct(df.value)).show() + +---------------------+ + |count(DISTINCT value)| + +---------------------+ + | 2| + +---------------------+ + + >>> df.select(sf.countDistinct(df.value)).show() + +---------------------+ + |count(DISTINCT value)| + +---------------------+ + | 2| + +---------------------+ """ - return _invoke_function_over_columns("octet_length", col) + return count_distinct(col, *cols) @_try_remote_functions -def bit_length(col: "ColumnOrName") -> Column: - """ - Calculates the bit length for the specified string column. +def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: + """Returns a new :class:`Column` for distinct count of ``col`` or ``cols``. - .. versionadded:: 3.3.0 + .. versionadded:: 3.2.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -7617,2903 +7385,2829 @@ def bit_length(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Source column or strings. - A column that evaluates to a string or binary. + first column to compute on. + cols : :class:`~pyspark.sql.Column` or column name + other columns to compute on. Returns ------- :class:`~pyspark.sql.Column` - Bit length of the col - Returns a column that evaluates to an integer. + distinct values of these two column values. + + See Also + -------- + :meth:`pyspark.sql.functions.approx_count_distinct` Examples -------- + Example 1: Counting distinct values of a single column + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) - >>> df.select('*', sf.bit_length('cat')).show() - +---+---------------+ - |cat|bit_length(cat)| - +---+---------------+ - |cat| 24| - | 🐈| 32| - +---+---------------+ + >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) + >>> df.select(sf.count_distinct(df.value)).show() + +---------------------+ + |count(DISTINCT value)| + +---------------------+ + | 2| + +---------------------+ + + Example 2: Counting distinct values of multiple columns + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) + >>> df.select(sf.count_distinct(df.value1, df.value2)).show() + +------------------------------+ + |count(DISTINCT value1, value2)| + +------------------------------+ + | 2| + +------------------------------+ + + Example 3: Counting distinct values with column names as strings + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) + >>> df.select(sf.count_distinct("value1", "value2")).show() + +------------------------------+ + |count(DISTINCT value1, value2)| + +------------------------------+ + | 2| + +------------------------------+ """ - return _invoke_function_over_columns("bit_length", col) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function( + "count_distinct", _to_java_column(col), _to_seq(sc, cols, _to_java_column) + ) @_try_remote_functions -def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: - """A function translate any character in the `srcCol` by a character in `matching`. - The characters in `replace` is corresponding to the characters in `matching`. - Translation will happen whenever any character in the string is matching with the character - in the `matching`. +def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: + """Aggregate function: returns the first value in a group. - .. versionadded:: 1.5.0 + The function by default returns the first values it sees. It will return the first non-null + value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + + .. versionadded:: 1.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Notes + ----- + The function is non-deterministic because its results depends on the order of the + rows which may be non-deterministic after a shuffle. + Parameters ---------- - srcCol : :class:`~pyspark.sql.Column` or column name - Source column or strings. - A column that evaluates to a string. - matching : str - matching characters. - A column that evaluates to a string. - replace : str - characters for replacement. If this is shorter than `matching` string then - those chars that don't have replacement will be dropped. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + column to fetch first value for. + A column of any type. + ignorenulls : bool + if first value is null then look for first non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - replaced value. - Returns a column that evaluates to a string. + first value of the group. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('translate',)], ['a']) - >>> df.select('*', sf.translate('a', "rnlt", "123")).show() - +---------+-----------------------+ - | a|translate(a, rnlt, 123)| - +---------+-----------------------+ - |translate| 1a2s3ae| - +---------+-----------------------+ + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) + >>> df = df.orderBy(df.age) + >>> df.groupby("name").agg(sf.first("age")).orderBy("name").show() + +-----+----------+ + | name|first(age)| + +-----+----------+ + |Alice| NULL| + | Bob| 5| + +-----+----------+ + + To ignore any null values, set ``ignorenulls`` to `True` + + >>> df.groupby("name").agg(sf.first("age", ignorenulls=True)).orderBy("name").show() + +-----+----------+ + | name|first(age)| + +-----+----------+ + |Alice| 2| + | Bob| 5| + +-----+----------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function( - "translate", _to_java_column(srcCol), _enum_to_value(matching), _enum_to_value(replace) - ) + return _invoke_function("first", _to_java_column(col), _enum_to_value(ignorenulls)) @_try_remote_functions -def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: +def grouping(col: "ColumnOrName") -> Column: """ - Converts the input `col` to a binary value based on the supplied `format`. - The `format` can be a case-insensitive string literal of "hex", "utf-8", "utf8", - or "base64". By default, the binary format for conversion is "hex" if - `format` is omitted. The function returns NULL if at least one of the - input parameters is NULL. + Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated + or not, returns 1 for aggregated or 0 for not aggregated in the result set. - .. versionadded:: 3.5.0 + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert binary values. - A column that evaluates to a string. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + column to check if it's aggregated. - See Also - -------- - :meth:`pyspark.sql.functions.try_to_binary` + Returns + ------- + :class:`~pyspark.sql.Column` + returns 1 for aggregated or 0 for not aggregated in the result set. Examples -------- - Example 1: Convert string to a binary with encoding specified - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("abc",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() - [Row(r=b'abc')] - - Example 2: Convert string to a timestamp without encoding specified - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("414243",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() - [Row(r=b'ABC')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) + >>> df.cube("name").agg(sf.grouping("name"), sf.sum("age")).orderBy("name").show() + +-----+--------------+--------+ + | name|grouping(name)|sum(age)| + +-----+--------------+--------+ + | NULL| 1| 7| + |Alice| 0| 2| + | Bob| 0| 5| + +-----+--------------+--------+ """ - if format is not None: - return _invoke_function_over_columns("to_binary", col, format) - else: - return _invoke_function_over_columns("to_binary", col) + return _invoke_function_over_columns("grouping", col) @_try_remote_functions -def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: +def grouping_id(*cols: "ColumnOrName") -> Column: """ - Convert `col` to a string based on the `format`. - Throws an exception if the conversion fails. The format can consist of the following - characters, case insensitive: - '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the - format string matches a sequence of digits in the input value, generating a result - string of the same length as the corresponding sequence in the format string. - The result string is left-padded with zeros if the 0/9 sequence comprises more digits - than the matching part of the decimal value, starts with 0, and is before the decimal - point. Otherwise, it is padded with spaces. - '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). - ',' or 'G': Specifies the position of the grouping (thousands) separator (,). - There must be a 0 or 9 to the left and right of each grouping separator. - '$': Specifies the location of the $ currency sign. This character may only be specified once. - 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at - the beginning or end of the format string). Note that 'S' prints '+' for positive - values but 'MI' prints a space. - 'PR': Only allowed at the end of the format string; specifies that the result string - will be wrapped by angle brackets if the input value is negative. - If `col` is a datetime, `format` shall be a valid datetime pattern, see - Patterns. - If `col` is a binary, it is converted to a string in one of the formats: - 'base64': a base 64 string. - 'hex': a string in the hexadecimal format. - 'utf-8': the input binary is decoded to UTF-8 string. + Aggregate function: returns the level of grouping, equals to - .. versionadded:: 3.5.0 + (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn) + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The list of columns should match with grouping columns exactly, or empty (means all + the grouping columns). Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The value to convert to a string. - A column that evaluates to a numeric, date, timestamp, time, or binary. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert char values. - A column that evaluates to a string. Must be a constant when ``col`` is numeric - or binary. + cols : :class:`~pyspark.sql.Column` or column name + columns to check for. + + Returns + ------- + :class:`~pyspark.sql.Column` + returns level of the grouping it relates to. Examples -------- - >>> df = spark.createDataFrame([(78.12,)], ["e"]) - >>> df.select(to_char(df.e, lit("$99.99")).alias('r')).collect() - [Row(r='$78.12')] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [(1, "a", "a"), (3, "a", "a"), (4, "b", "c")], ["c1", "c2", "c3"]) + >>> df.cube("c2", "c3").agg(sf.grouping_id(), sf.sum("c1")).orderBy("c2", "c3").show() + +----+----+-------------+-------+ + | c2| c3|grouping_id()|sum(c1)| + +----+----+-------------+-------+ + |NULL|NULL| 3| 8| + |NULL| a| 2| 4| + |NULL| c| 2| 4| + | a|NULL| 1| 4| + | a| a| 0| 4| + | b|NULL| 1| 4| + | b| c| 0| 4| + +----+----+-------------+-------+ """ - return _invoke_function_over_columns("to_char", col, format) + return _invoke_function_over_seq_of_columns("grouping_id", cols) @_try_remote_functions -def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: +def count_min_sketch( + col: "ColumnOrName", + eps: Union[Column, float], + confidence: Union[Column, float], + seed: Optional[Union[Column, int]] = None, +) -> Column: """ - Convert `col` to a string based on the `format`. - Throws an exception if the conversion fails. The format can consist of the following - characters, case insensitive: - '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the - format string matches a sequence of digits in the input value, generating a result - string of the same length as the corresponding sequence in the format string. - The result string is left-padded with zeros if the 0/9 sequence comprises more digits - than the matching part of the decimal value, starts with 0, and is before the decimal - point. Otherwise, it is padded with spaces. - '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). - ',' or 'G': Specifies the position of the grouping (thousands) separator (,). - There must be a 0 or 9 to the left and right of each grouping separator. - '$': Specifies the location of the $ currency sign. This character may only be specified once. - 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at - the beginning or end of the format string). Note that 'S' prints '+' for positive - values but 'MI' prints a space. - 'PR': Only allowed at the end of the format string; specifies that the result string - will be wrapped by angle brackets if the input value is negative. - If `col` is a datetime, `format` shall be a valid datetime pattern, see - Patterns. - If `col` is a binary, it is converted to a string in one of the formats: - 'base64': a base 64 string. - 'hex': a string in the hexadecimal format. - 'utf-8': the input binary is decoded to UTF-8 string. + Returns a count-min sketch of a column with the given esp, confidence and seed. + The result is an array of bytes, which can be deserialized to a `CountMinSketch` before usage. + Count-min sketch is a probabilistic data structure used for cardinality estimation + using sub-linear space. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The value to convert to a string. - A column that evaluates to a numeric, date, timestamp, time, or binary. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert char values. - A column that evaluates to a string. Must be a constant when ``col`` is numeric - or binary. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + eps : :class:`~pyspark.sql.Column` or float + relative error, must be positive - Examples - -------- - >>> df = spark.createDataFrame([(78.12,)], ["e"]) - >>> df.select(to_varchar(df.e, lit("$99.99")).alias('r')).collect() - [Row(r='$78.12')] - """ - return _invoke_function_over_columns("to_varchar", col, format) + .. versionchanged:: 4.0.0 + `eps` now accepts float value. + confidence : :class:`~pyspark.sql.Column` or float + confidence, must be positive and less than 1.0 -@_try_remote_functions -def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: - """ - Convert string 'col' to a number based on the string format 'format'. - Throws an exception if the conversion fails. The format can consist of the following - characters, case insensitive: - '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the - format string matches a sequence of digits in the input string. If the 0/9 - sequence starts with 0 and is before the decimal point, it can only match a digit - sequence of the same size. Otherwise, if the sequence starts with 9 or is after - the decimal point, it can match a digit sequence that has the same or smaller size. - '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). - ',' or 'G': Specifies the position of the grouping (thousands) separator (,). - There must be a 0 or 9 to the left and right of each grouping separator. - 'col' must match the grouping separator relevant for the size of the number. - '$': Specifies the location of the $ currency sign. This character may only be - specified once. - 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed - once at the beginning or end of the format string). Note that 'S' allows '-' - but 'MI' does not. - 'PR': Only allowed at the end of the format string; specifies that 'col' indicates a - negative number with wrapping angled brackets. + .. versionchanged:: 4.0.0 + `confidence` now accepts float value. - .. versionadded:: 3.5.0 + seed : :class:`~pyspark.sql.Column` or int, optional + random seed - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert number values. - A column that evaluates to a string. Must be a constant. + .. versionchanged:: 4.0.0 + `seed` now accepts int value. - See Also - -------- - :meth:`pyspark.sql.functions.try_to_number` + Returns + ------- + :class:`~pyspark.sql.Column` + count-min sketch of the column Examples -------- - >>> df = spark.createDataFrame([("$78.12",)], ["e"]) - >>> df.select(to_number(df.e, lit("$99.99")).alias('r')).collect() - [Row(r=Decimal('78.12'))] - """ - return _invoke_function_over_columns("to_number", col, format) + Example 1: Using columns as arguments + >>> from pyspark.sql import functions as sf + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch(sf.col("id"), sf.lit(3.0), sf.lit(0.1), sf.lit(1))) + ... ).show(truncate=False) + +------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 3.0, 0.1, 1)) | + +------------------------------------------------------------------------+ + |0000000100000000000000640000000100000001000000005D8D6AB90000000000000064| + +------------------------------------------------------------------------+ -@_try_remote_functions -def replace( - src: "ColumnOrName", search: "ColumnOrName", replace: Optional["ColumnOrName"] = None -) -> Column: - """ - Replaces all occurrences of `search` with `replace`. + Example 2: Using numbers as arguments - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch("id", 1.0, 0.3, 2)) + ... ).show(truncate=False) + +----------------------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 1.0, 0.3, 2)) | + +----------------------------------------------------------------------------------------+ + |0000000100000000000000640000000100000002000000005D96391C00000000000000320000000000000032| + +----------------------------------------------------------------------------------------+ - Parameters - ---------- - src : :class:`~pyspark.sql.Column` or str - A column of string to be replaced. - A column that evaluates to a string. - search : :class:`~pyspark.sql.Column` or str - A column of string, If `search` is not found in `str`, `str` is returned unchanged. - A column that evaluates to a string. - replace : :class:`~pyspark.sql.Column` or str, optional - A column of string, If `replace` is not specified or is an empty string, - nothing replaces the string that is removed from `str`. - A column that evaluates to a string. + Example 3: Using a long seed - Examples - -------- - >>> df = spark.createDataFrame([("ABCabc", "abc", "DEF",)], ["a", "b", "c"]) - >>> df.select(replace(df.a, df.b, df.c).alias('r')).collect() - [Row(r='ABCDEF')] + >>> from pyspark.sql import functions as sf + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.2, 1111111111111111111)) + ... ).show(truncate=False) + +----------------------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 1.5, 0.2, 1111111111111111111)) | + +----------------------------------------------------------------------------------------+ + |00000001000000000000006400000001000000020000000044078BA100000000000000320000000000000032| + +----------------------------------------------------------------------------------------+ - >>> df.select(replace(df.a, df.b).alias('r')).collect() - [Row(r='ABC')] + Example 4: Using a random seed + + >>> from pyspark.sql import functions as sf + >>> spark.range(100).select( + ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.6)) + ... ).show(truncate=False) # doctest: +SKIP + +----------------------------------------------------------------------------------------------------------------------------------------+ + |hex(count_min_sketch(id, 1.5, 0.6, 2120704260)) | + +----------------------------------------------------------------------------------------------------------------------------------------+ + |0000000100000000000000640000000200000002000000005ADECCEE00000000153EBE090000000000000033000000000000003100000000000000320000000000000032| + +----------------------------------------------------------------------------------------------------------------------------------------+ """ - if replace is not None: - return _invoke_function_over_columns("replace", src, search, replace) + _eps = lit(eps) + _conf = lit(confidence) + if seed is None: + return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf) else: - return _invoke_function_over_columns("replace", src, search) + return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf, lit(seed)) @_try_remote_functions -def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnOrName") -> Column: +def input_file_name() -> Column: """ - Splits `str` by delimiter and return requested part of the split (1-based). - If any input is null, returns null. if `partNum` is out of range of split parts, - returns empty string. If `partNum` is 0, throws an error. If `partNum` is negative, - the parts are counted backward from the end of the string. - If the `delimiter` is an empty string, the `str` is not split. + Creates a string column for the file name of the current Spark task. - .. versionadded:: 3.5.0 + .. versionadded:: 1.6.0 - Parameters - ---------- - src : :class:`~pyspark.sql.Column` or column name - A column of string to be split. - A column that evaluates to a string. - delimiter : :class:`~pyspark.sql.Column` or column name - A column of string, the delimiter used for split. - A column that evaluates to a string. - partNum : :class:`~pyspark.sql.Column` or column name - The requested part of the split (1-based). - A column that evaluates to an integer. + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Returns + ------- + :class:`~pyspark.sql.Column` + file names. See Also -------- - :meth:`pyspark.sql.functions.sentences` - :meth:`pyspark.sql.functions.split` + :meth:`pyspark.sql.functions.input_file_block_length` + :meth:`pyspark.sql.functions.input_file_block_start` Examples -------- + >>> import os >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("11.12.13", ".", 3,)], ["a", "b", "c"]) - >>> df.select("*", sf.split_part("a", "b", "c")).show() - +--------+---+---+-------------------+ - | a| b| c|split_part(a, b, c)| - +--------+---+---+-------------------+ - |11.12.13| .| 3| 13| - +--------+---+---+-------------------+ - - >>> df.select("*", sf.split_part(df.a, df.b, sf.lit(-2))).show() - +--------+---+---+--------------------+ - | a| b| c|split_part(a, b, -2)| - +--------+---+---+--------------------+ - |11.12.13| .| 3| 12| - +--------+---+---+--------------------+ + >>> path = os.path.abspath(__file__) + >>> df = spark.read.text(path) + >>> df.select(sf.input_file_name()).first() + Row(input_file_name()='file:///...') """ - return _invoke_function_over_columns("split_part", src, delimiter, partNum) + return _invoke_function("input_file_name") @_try_remote_functions -def substr( - str: "ColumnOrName", pos: "ColumnOrName", len: Optional["ColumnOrName"] = None -) -> Column: - """ - Returns the substring of `str` that starts at `pos` and is of length `len`, - or the slice of byte array that starts at `pos` and is of length `len`. +def isnan(col: "ColumnOrName") -> Column: + """An expression that returns true if the column is NaN. - .. versionadded:: 3.5.0 + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or column name - A column of string. - A column that evaluates to a string or binary. - pos : :class:`~pyspark.sql.Column` or column name - The starting position of the substring. - A column that evaluates to an integer. - len : :class:`~pyspark.sql.Column` or column name, optional - The length of the substring. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a double or float. Returns ------- :class:`~pyspark.sql.Column` - substring of given value. - Returns a column of the same type as the input. + True if value is NaN and False otherwise. + Returns a column that evaluates to a boolean. See Also -------- - :meth:`pyspark.sql.functions.instr` - :meth:`pyspark.sql.functions.substring` - :meth:`pyspark.sql.functions.substring_index` - :meth:`pyspark.sql.Column.substr` - :meth:`pyspark.sql.functions.locate` + :meth:`pyspark.sql.functions.isnull` + :meth:`pyspark.sql.functions.isnotnull` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Spark SQL", 5, 1,)], ["a", "b", "c"]) - >>> df.select("*", sf.substr("a", "b", "c")).show() - +---------+---+---+---------------+ - | a| b| c|substr(a, b, c)| - +---------+---+---+---------------+ - |Spark SQL| 5| 1| k| - +---------+---+---+---------------+ - - >>> df.select("*", sf.substr(df.a, df.b)).show() - +---------+---+---+------------------------+ - | a| b| c|substr(a, b, 2147483647)| - +---------+---+---+------------------------+ - |Spark SQL| 5| 1| k SQL| - +---------+---+---+------------------------+ + >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) + >>> df.select("*", sf.isnan("a"), sf.isnan(df.b)).show() + +---+---+--------+--------+ + | a| b|isnan(a)|isnan(b)| + +---+---+--------+--------+ + |1.0|NaN| false| true| + |NaN|2.0| true| false| + +---+---+--------+--------+ """ - if len is not None: - return _invoke_function_over_columns("substr", str, pos, len) - else: - return _invoke_function_over_columns("substr", str, pos) + return _invoke_function_over_columns("isnan", col) @_try_remote_functions -def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: - """ - Formats the arguments in printf-style and returns the result as a string column. +def isnull(col: "ColumnOrName") -> Column: + """An expression that returns true if the column is null. - .. versionadded:: 3.5.0 + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - format : :class:`~pyspark.sql.Column` or str - string that can contain embedded format tags and used as result column's value. - A column that evaluates to a string. - cols : :class:`~pyspark.sql.Column` or str - column names or :class:`~pyspark.sql.Column`\\s to be used in formatting - Each a column of any type. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column of any type. + + Returns + ------- + :class:`~pyspark.sql.Column` + True if value is null and False otherwise. + Returns a column that evaluates to a boolean. See Also -------- - :meth:`pyspark.sql.functions.format_string` + :meth:`pyspark.sql.functions.isnan` + :meth:`pyspark.sql.functions.isnotnull` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("aa%d%s", 123, "cc",)], ["a", "b", "c"] - ... ).select(sf.printf("a", "b", "c")).show() - +---------------+ - |printf(a, b, c)| - +---------------+ - | aa123cc| - +---------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, None), (None, 2)], ("a", "b")) + >>> df.select("*", sf.isnull("a"), isnull(df.b)).show() + +----+----+-----------+-----------+ + | a| b|(a IS NULL)|(b IS NULL)| + +----+----+-----------+-----------+ + | 1|NULL| false| true| + |NULL| 2| true| false| + +----+----+-----------+-----------+ """ - from pyspark.sql.classic.column import _to_java_column, _to_seq - - sc = _get_active_spark_context() - return _invoke_function("printf", _to_java_column(format), _to_seq(sc, cols, _to_java_column)) + return _invoke_function_over_columns("isnull", col) @_try_remote_functions -def position( - substr: "ColumnOrName", str: "ColumnOrName", start: Optional["ColumnOrName"] = None -) -> Column: - """ - Returns the position of the first occurrence of `substr` in `str` after position `start`. - The given `start` and return value are 1-based. +def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: + """Aggregate function: returns the last value in a group. - .. versionadded:: 3.5.0 + The function by default returns the last values it sees. It will return the last non-null + value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + + .. versionadded:: 1.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The function is non-deterministic because its results depends on the order of the + rows which may be non-deterministic after a shuffle. Parameters ---------- - substr : :class:`~pyspark.sql.Column` or str - A column of string, substring. - A column that evaluates to a string. - str : :class:`~pyspark.sql.Column` or str - A column of string. - A column that evaluates to a string. - start : :class:`~pyspark.sql.Column` or str, optional - The start position. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + column to fetch last value for. + A column of any type. + ignorenulls : bool + if last value is null then look for non-null value. ``False`` by default. + A column that evaluates to a boolean. Must be a constant. + + Returns + ------- + :class:`~pyspark.sql.Column` + last value of the group. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] - ... ).select(sf.position("a", "b", "c")).show() - +-----------------+ - |position(a, b, c)| - +-----------------+ - | 7| - +-----------------+ - - >>> spark.createDataFrame( - ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] - ... ).select(sf.position("a", "b")).show() - +-----------------+ - |position(a, b, 1)| - +-----------------+ - | 4| - +-----------------+ - """ - if start is not None: - return _invoke_function_over_columns("position", substr, str, start) - else: - return _invoke_function_over_columns("position", substr, str) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) + >>> df = df.orderBy(df.age.desc()) + >>> df.groupby("name").agg(sf.last("age")).orderBy("name").show() + +-----+---------+ + | name|last(age)| + +-----+---------+ + |Alice| NULL| + | Bob| 5| + +-----+---------+ + To ignore any null values, set ``ignorenulls`` to `True` -@_try_remote_functions -def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: + >>> df.groupby("name").agg(sf.last("age", ignorenulls=True)).orderBy("name").show() + +-----+---------+ + | name|last(age)| + +-----+---------+ + |Alice| 2| + | Bob| 5| + +-----+---------+ """ - Returns a boolean. The value is True if str ends with suffix. - Returns NULL if either input expression is NULL. Otherwise, returns False. - Both str or suffix must be of STRING or BINARY type. + from pyspark.sql.classic.column import _to_java_column - .. versionadded:: 3.5.0 + return _invoke_function("last", _to_java_column(col), _enum_to_value(ignorenulls)) - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - The input value to test. - A column that evaluates to a string or binary. - suffix : :class:`~pyspark.sql.Column` or str - The suffix to test for. - A column that evaluates to a string or binary. - Examples - -------- - >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) - >>> df.select(endswith(df.a, df.b).alias('r')).collect() - [Row(r=False)] +@_try_remote_functions +def monotonically_increasing_id() -> Column: + """A column that generates monotonically increasing 64-bit integers. - >>> df = spark.createDataFrame([("414243", "4243",)], ["e", "f"]) - >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) - >>> df.printSchema() - root - |-- e: binary (nullable = true) - |-- f: binary (nullable = true) - >>> df.select(endswith("e", "f"), endswith("f", "e")).show() - +--------------+--------------+ - |endswith(e, f)|endswith(f, e)| - +--------------+--------------+ - | true| false| - +--------------+--------------+ - """ - return _invoke_function_over_columns("endswith", str, suffix) + The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. + The current implementation puts the partition ID in the upper 31 bits, and the record number + within each partition in the lower 33 bits. The assumption is that the data frame has + less than 1 billion partitions, and each partition has less than 8 billion records. + .. versionadded:: 1.6.0 -@_try_remote_functions -def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: - """ - Returns a boolean. The value is True if str starts with prefix. - Returns NULL if either input expression is NULL. Otherwise, returns False. - Both str or prefix must be of STRING or BINARY type. + .. versionchanged:: 3.4.0 + Supports Spark Connect. - .. versionadded:: 3.5.0 + Notes + ----- + The function is non-deterministic because its result depends on partition IDs. - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - The input value to test. - A column that evaluates to a string or binary. - prefix : :class:`~pyspark.sql.Column` or str - The prefix to test for. - A column that evaluates to a string or binary. + As an example, consider a :class:`DataFrame` with two partitions, each with 3 records. + This expression would return the following IDs: + 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. + + Returns + ------- + :class:`~pyspark.sql.Column` + last value of the group. Examples -------- - >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) - >>> df.select(startswith(df.a, df.b).alias('r')).collect() - [Row(r=True)] - - >>> df = spark.createDataFrame([("414243", "4142",)], ["e", "f"]) - >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) - >>> df.printSchema() - root - |-- e: binary (nullable = true) - |-- f: binary (nullable = true) - >>> df.select(startswith("e", "f"), startswith("f", "e")).show() - +----------------+----------------+ - |startswith(e, f)|startswith(f, e)| - +----------------+----------------+ - | true| false| - +----------------+----------------+ + >>> from pyspark.sql import functions as sf + >>> spark.range(0, 10, 1, 2).select( + ... "*", + ... sf.spark_partition_id(), + ... sf.monotonically_increasing_id()).show() + +---+--------------------+-----------------------------+ + | id|SPARK_PARTITION_ID()|monotonically_increasing_id()| + +---+--------------------+-----------------------------+ + | 0| 0| 0| + | 1| 0| 1| + | 2| 0| 2| + | 3| 0| 3| + | 4| 0| 4| + | 5| 1| 8589934592| + | 6| 1| 8589934593| + | 7| 1| 8589934594| + | 8| 1| 8589934595| + | 9| 1| 8589934596| + +---+--------------------+-----------------------------+ """ - return _invoke_function_over_columns("startswith", str, prefix) + return _invoke_function("monotonically_increasing_id") @_try_remote_functions -def char(col: "ColumnOrName") -> Column: - """ - Returns the ASCII character having the binary equivalent to `col`. If col is larger than 256 the - result is equivalent to char(col % 256) +def nanvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """Returns col1 if it is not NaN, or col2 if col1 is NaN. - .. versionadded:: 3.5.0 + Both inputs should be floating point columns (:class:`DoubleType` or :class:`FloatType`). + + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a long. + col1 : :class:`~pyspark.sql.Column` or column name + first column to check. + A column that evaluates to a double or float. + col2 : :class:`~pyspark.sql.Column` or column name + second column to return if first is NaN. + A column that evaluates to a double or float. + + Returns + ------- + :class:`~pyspark.sql.Column` + value from first column or second if first is NaN . + Returns a column of the same type as the first input. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.char(sf.lit(65))).show() - +--------+ - |char(65)| - +--------+ - | A| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1.0, float('nan')), (float('nan'), 2.0)], ("a", "b")) + >>> df.select("*", sf.nanvl("a", "b"), sf.nanvl(df.a, df.b)).show() + +---+---+-----------+-----------+ + | a| b|nanvl(a, b)|nanvl(a, b)| + +---+---+-----------+-----------+ + |1.0|NaN| 1.0| 1.0| + |NaN|2.0| 2.0| 2.0| + +---+---+-----------+-----------+ """ - return _invoke_function_over_columns("char", col) + return _invoke_function_over_columns("nanvl", col1, col2) @_try_remote_functions -def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: - """ - Remove the leading and trailing `trim` characters from `str`. +def percentile( + col: "ColumnOrName", + percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], + frequency: Union[Column, int] = 1, +) -> Column: + """Returns the exact percentile(s) of numeric column `expr` at the given percentage(s) + with value range in [0.0, 1.0]. .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - trim : :class:`~pyspark.sql.Column` or str, optional - The trim string characters to trim, the default value is a single space. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats + percentage in decimal (must be between 0.0 and 1.0). + frequency : :class:`~pyspark.sql.Column` or int is a positive numeric literal which + controls frequency. + + Returns + ------- + :class:`~pyspark.sql.Column` + the exact `percentile` of the numeric column. + + See Also + -------- + :meth:`pyspark.sql.functions.median` + :meth:`pyspark.sql.functions.approx_percentile` + :meth:`pyspark.sql.functions.percentile_approx` Examples -------- - >>> df = spark.createDataFrame([("SSparkSQLS", "SL", )], ['a', 'b']) - >>> df.select(btrim(df.a, df.b).alias('r')).collect() - [Row(r='parkSQ')] + >>> from pyspark.sql import functions as sf + >>> key = (sf.col("id") % 3).alias("key") + >>> value = (sf.randn(42) + key * 10).alias("value") + >>> df = spark.range(0, 1000, 1, 1).select(key, value) + >>> df.select( + ... sf.percentile("value", [0.25, 0.5, 0.75], sf.lit(1)) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |percentile(value, array(0.25, 0.5, 0.75), 1) | + +--------------------------------------------------------+ + |[0.7441991494121..., 9.9900713756..., 19.33740203080...]| + +--------------------------------------------------------+ - >>> df = spark.createDataFrame([(" SparkSQL ",)], ['a']) - >>> df.select(btrim(df.a).alias('r')).collect() - [Row(r='SparkSQL')] + >>> df.groupBy("key").agg( + ... sf.percentile("value", sf.lit(0.5), sf.lit(1)) + ... ).sort("key").show() + +---+-------------------------+ + |key|percentile(value, 0.5, 1)| + +---+-------------------------+ + | 0| -0.03449962216667901| + | 1| 9.990389751837329| + | 2| 19.967859769284075| + +---+-------------------------+ """ - if trim is not None: - return _invoke_function_over_columns("btrim", str, trim) - else: - return _invoke_function_over_columns("btrim", str) + percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) + return _invoke_function_over_columns("percentile", col, percentage, lit(frequency)) @_try_remote_functions -def char_length(str: "ColumnOrName") -> Column: - """ - Returns the character length of string data or number of bytes of binary data. - The length of string data includes the trailing spaces. - The length of binary data includes binary zeros. +def percentile_approx( + col: "ColumnOrName", + percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], + accuracy: Union[Column, int] = 10000, +) -> Column: + """Returns the approximate `percentile` of the numeric column `col` which is the smallest value + in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` + of `col` values is less than the value or equal to that value. - .. versionadded:: 3.5.0 + + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string or binary. + col : :class:`~pyspark.sql.Column` or column name + input column. + percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats + percentage in decimal (must be between 0.0 and 1.0). + When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. + In this case, returns the approximate percentile array of column col + at the given percentage array. + accuracy : :class:`~pyspark.sql.Column` or int + is a positive numeric literal which controls approximation accuracy + at the cost of memory. Higher value of accuracy yields better accuracy, + 1.0/accuracy is the relative error of the approximation. (default: 10000). + + Returns + ------- + :class:`~pyspark.sql.Column` + approximate `percentile` of the numeric column. See Also -------- - :meth:`pyspark.sql.functions.character_length` - :meth:`pyspark.sql.functions.length` + :meth:`pyspark.sql.functions.median` + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.approx_percentile` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.char_length(sf.lit("SparkSQL"))).show() - +---------------------+ - |char_length(SparkSQL)| - +---------------------+ - | 8| - +---------------------+ + >>> from pyspark.sql import functions as sf + >>> key = (sf.col("id") % 3).alias("key") + >>> value = (sf.randn(42) + key * 10).alias("value") + >>> df = spark.range(0, 1000, 1, 1).select(key, value) + >>> df.select( + ... sf.percentile_approx("value", [0.25, 0.5, 0.75], 1000000) + ... ).show(truncate=False) + +----------------------------------------------------------+ + |percentile_approx(value, array(0.25, 0.5, 0.75), 1000000) | + +----------------------------------------------------------+ + |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| + +----------------------------------------------------------+ + + >>> df.groupBy("key").agg( + ... sf.percentile_approx("value", sf.lit(0.5), sf.lit(1000000)) + ... ).sort("key").show() + +---+--------------------------------------+ + |key|percentile_approx(value, 0.5, 1000000)| + +---+--------------------------------------+ + | 0| -0.03519435193070...| + | 1| 9.990389751837...| + | 2| 19.967859769284...| + +---+--------------------------------------+ """ - return _invoke_function_over_columns("char_length", str) + percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) + return _invoke_function_over_columns("percentile_approx", col, percentage, lit(accuracy)) @_try_remote_functions -def character_length(str: "ColumnOrName") -> Column: - """ - Returns the character length of string data or number of bytes of binary data. - The length of string data includes the trailing spaces. - The length of binary data includes binary zeros. +def approx_percentile( + col: "ColumnOrName", + percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], + accuracy: Union[Column, int] = 10000, +) -> Column: + """Returns the approximate `percentile` of the numeric column `col` which is the smallest value + in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` + of `col` values is less than the value or equal to that value. .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string or binary. + col : :class:`~pyspark.sql.Column` or column name + input column. + percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats + percentage in decimal (must be between 0.0 and 1.0). + When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. + In this case, returns the approximate percentile array of column col + at the given percentage array. + accuracy : :class:`~pyspark.sql.Column` or int + is a positive numeric literal which controls approximation accuracy + at the cost of memory. Higher value of accuracy yields better accuracy, + 1.0/accuracy is the relative error of the approximation. (default: 10000). + + Returns + ------- + :class:`~pyspark.sql.Column` + approximate `percentile` of the numeric column. See Also -------- - :meth:`pyspark.sql.functions.char_length` - :meth:`pyspark.sql.functions.length` + :meth:`pyspark.sql.functions.median` + :meth:`pyspark.sql.functions.percentile` + :meth:`pyspark.sql.functions.percentile_approx` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.character_length(sf.lit("SparkSQL"))).show() - +--------------------------+ - |character_length(SparkSQL)| - +--------------------------+ - | 8| - +--------------------------+ + >>> from pyspark.sql import functions as sf + >>> key = (sf.col("id") % 3).alias("key") + >>> value = (sf.randn(42) + key * 10).alias("value") + >>> df = spark.range(0, 1000, 1, 1).select(key, value) + >>> df.select( + ... sf.approx_percentile("value", [0.25, 0.5, 0.75], 1000000) + ... ).show(truncate=False) + +----------------------------------------------------------+ + |approx_percentile(value, array(0.25, 0.5, 0.75), 1000000) | + +----------------------------------------------------------+ + |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| + +----------------------------------------------------------+ + + >>> df.groupBy("key").agg( + ... sf.approx_percentile("value", sf.lit(0.5), sf.lit(1000000)) + ... ).sort("key").show() + +---+--------------------------------------+ + |key|approx_percentile(value, 0.5, 1000000)| + +---+--------------------------------------+ + | 0| -0.03519435193070...| + | 1| 9.990389751837...| + | 2| 19.967859769284...| + +---+--------------------------------------+ """ - return _invoke_function_over_columns("character_length", str) + percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) + return _invoke_function_over_columns("approx_percentile", col, percentage, lit(accuracy)) @_try_remote_functions -def chr(n: "ColumnOrName") -> Column: - """ - Returns the ASCII character having the binary equivalent to `n`. - If n is larger than 256 the result is equivalent to chr(n % 256). +def rand(seed: Optional[int] = None) -> Column: + """Generates a random column with independent and identically distributed (i.i.d.) samples + uniformly distributed in [0.0, 1.0). - .. versionadded:: 4.1.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The function is non-deterministic in general case. Parameters ---------- - n : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a long. + seed : int, optional + Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. + + Returns + ------- + :class:`~pyspark.sql.Column` + A column of random values. + Returns a column that evaluates to a double. + + See Also + -------- + :meth:`pyspark.sql.functions.randn` + :meth:`pyspark.sql.functions.randstr` + :meth:`pyspark.sql.functions.uniform` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(60, 70).select("*", sf.chr("id")).show() - +---+-------+ - | id|chr(id)| - +---+-------+ - | 60| <| - | 61| =| - | 62| >| - | 63| ?| - | 64| @| - | 65| A| - | 66| B| - | 67| C| - | 68| D| - | 69| E| - +---+-------+ + Example 1: Generate a random column without a seed + + >>> from pyspark.sql import functions as sf + >>> spark.range(0, 2, 1, 1).select("*", sf.rand()).show() # doctest: +SKIP + +---+-------------------------+ + | id|rand(-158884697681280011)| + +---+-------------------------+ + | 0| 0.9253464547887...| + | 1| 0.6533254118758...| + +---+-------------------------+ + + Example 2: Generate a random column with a specific seed + + >>> spark.range(0, 2, 1, 1).select("*", sf.rand(seed=42)).show() + +---+------------------+ + | id| rand(42)| + +---+------------------+ + | 0| 0.619189370225...| + | 1|0.5096018842446...| + +---+------------------+ """ - return _invoke_function_over_columns("chr", n) + if seed is not None: + return _invoke_function("rand", _enum_to_value(seed)) + else: + return _invoke_function("rand") + + +random = rand @_try_remote_functions -def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: - """ - This is a special version of `to_binary` that performs the same operation, but returns a NULL - value instead of raising an error if the conversion cannot be performed. +def randn(seed: Optional[int] = None) -> Column: + """Generates a random column with independent and identically distributed (i.i.d.) samples + from the standard normal distribution. - .. versionadded:: 3.5.0 + .. versionadded:: 1.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The function is non-deterministic in general case. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert binary values. - A column that evaluates to a string. Must be a constant. + seed : int (default: None) + Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. + + Returns + ------- + :class:`~pyspark.sql.Column` + A column of random values. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.to_binary` + :meth:`pyspark.sql.functions.rand` + :meth:`pyspark.sql.functions.randstr` + :meth:`pyspark.sql.functions.uniform` Examples -------- - Example 1: Convert string to a binary with encoding specified + Example 1: Generate a random column without a seed - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("abc",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() - [Row(r=b'abc')] + >>> from pyspark.sql import functions as sf + >>> spark.range(0, 2, 1, 1).select("*", sf.randn()).show() # doctest: +SKIP + +---+--------------------------+ + | id|randn(3968742514375399317)| + +---+--------------------------+ + | 0| -0.47968645355788...| + | 1| -0.4950952457305...| + +---+--------------------------+ - Example 2: Convert string to a timestamp without encoding specified - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("414243",)], ["e"]) - >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() - [Row(r=b'ABC')] - - Example 3: Converion failure results in NULL when ANSI mode is on + Example 2: Generate a random column with a specific seed - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.range(1) - ... df.select(sf.try_to_binary(sf.lit("malformed"), sf.lit("hex"))).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +-----------------------------+ - |try_to_binary(malformed, hex)| - +-----------------------------+ - | NULL| - +-----------------------------+ + >>> spark.range(0, 2, 1, 1).select("*", sf.randn(seed=42)).show() + +---+------------------+ + | id| randn(42)| + +---+------------------+ + | 0| 2.384479054241...| + | 1|0.1920934041293...| + +---+------------------+ """ - if format is not None: - return _invoke_function_over_columns("try_to_binary", col, format) + if seed is not None: + return _invoke_function("randn", _enum_to_value(seed)) else: - return _invoke_function_over_columns("try_to_binary", col) + return _invoke_function("randn") @_try_remote_functions -def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: +def round(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: """ - Convert string 'col' to a number based on the string format `format`. Returns NULL if the - string 'col' does not match the expected format. The format follows the same semantics as the - to_number function. + Round the given value to `scale` decimal places using HALF_UP rounding mode if `scale` >= 0 + or at integral part when `scale` < 0. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - format : :class:`~pyspark.sql.Column` or str, optional - format to use to convert number values. - A column that evaluates to a string. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + The target column or column name to compute the round on. + A column that evaluates to a numeric. + scale : :class:`~pyspark.sql.Column` or int, optional + An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. - See Also - -------- - :meth:`pyspark.sql.functions.to_number` + .. versionchanged:: 4.0.0 + Support Column type. + + Returns + ------- + :class:`~pyspark.sql.Column` + A column for the rounded value. + Returns a column of the same type as the input. Examples -------- - Example 1: Convert a string to a number with a format specified + Example 1: Compute the rounded of a column value >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("$78.12",)], ["e"]) - >>> df.select(sf.try_to_number(df.e, sf.lit("$99.99")).alias('r')).show() - +-----+ - | r| - +-----+ - |78.12| - +-----+ + >>> spark.range(1).select(sf.round(sf.lit(2.5))).show() + +-------------+ + |round(2.5, 0)| + +-------------+ + | 3.0| + +-------------+ - Example 2: Converion failure results in NULL when ANSI mode is on + Example 2: Compute the rounded of a column value with a specified scale >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.range(1) - ... df.select(sf.try_to_number(sf.lit("77"), sf.lit("$99.99")).alias('r')).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +----+ - | r| - +----+ - |NULL| - +----+ + >>> spark.range(1).select(sf.round(sf.lit(2.1267), sf.lit(2))).show() + +----------------+ + |round(2.1267, 2)| + +----------------+ + | 2.13| + +----------------+ """ - return _invoke_function_over_columns("try_to_number", col, format) + if scale is None: + return _invoke_function_over_columns("round", col) + else: + scale = _enum_to_value(scale) + scale = lit(scale) if isinstance(scale, int) else scale + return _invoke_function_over_columns("round", col, scale) @_try_remote_functions -def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def truncate(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: """ - Returns a boolean. The value is True if right is found inside left. - Returns NULL if either input expression is NULL. Otherwise, returns False. - Both left or right must be of STRING or BINARY type. + Truncate the given value toward zero to `scale` decimal places when `scale` >= 0, + or to the left of the decimal point when `scale` < 0. `scale` defaults to 0. - .. versionadded:: 3.5.0 + Unlike :func:`round`, the result is always rounded toward zero, and unlike :func:`floor` + negative values are not rounded toward negative infinity. + + .. versionadded:: 4.4.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or str - The input to check; may be NULL. - A column that evaluates to a string or binary. - right : :class:`~pyspark.sql.Column` or str - The value to find; may be NULL. - A column that evaluates to a string or binary. + col : :class:`~pyspark.sql.Column` or column name + The target column or column name to truncate. + A column that evaluates to a numeric. + scale : :class:`~pyspark.sql.Column` or int, optional + An optional parameter to control the number of decimal places to keep. + A column that evaluates to an integer. Must be a constant. Defaults to 0. + + Returns + ------- + :class:`~pyspark.sql.Column` + A column for the truncated value, of the same type as the input, except that a decimal + input may return a decimal of different precision and scale. + + See Also + -------- + :meth:`pyspark.sql.functions.round` + :meth:`pyspark.sql.functions.trunc` + :meth:`pyspark.sql.functions.floor` + :meth:`pyspark.sql.functions.ceil` Examples -------- - >>> df = spark.createDataFrame([("Spark SQL", "Spark")], ['a', 'b']) - >>> df.select(contains(df.a, df.b).alias('r')).collect() - [Row(r=True)] + Example 1: Truncate toward zero to a given number of decimal places - >>> df = spark.createDataFrame([("414243", "4243",)], ["c", "d"]) - >>> df = df.select(to_binary("c").alias("c"), to_binary("d").alias("d")) - >>> df.printSchema() - root - |-- c: binary (nullable = true) - |-- d: binary (nullable = true) - >>> df.select(contains("c", "d"), contains("d", "c")).show() - +--------------+--------------+ - |contains(c, d)|contains(d, c)| - +--------------+--------------+ - | true| false| - +--------------+--------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.truncate(sf.lit(15.79), sf.lit(1)).alias("r")).collect() + [Row(r=15.7)] + + Example 2: Truncation rounds toward zero for negative values + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.truncate(sf.lit(-2.99), sf.lit(0)).alias("r")).collect() + [Row(r=-2.0)] + + Example 3: The scale argument defaults to 0 when omitted + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.truncate(sf.lit(1234.5678)).alias("r")).collect() + [Row(r=1234.0)] """ - return _invoke_function_over_columns("contains", left, right) + if scale is None: + return _invoke_function_over_columns("truncate", col) + else: + scale = _enum_to_value(scale) + scale = lit(scale) if isinstance(scale, int) else scale + return _invoke_function_over_columns("truncate", col, scale) @_try_remote_functions -def elt(*inputs: "ColumnOrName") -> Column: +def bround(col: "ColumnOrName", scale: Optional[Union[Column, int]] = None) -> Column: """ - Returns the `n`-th input, e.g., returns `input2` when `n` is 2. - The function returns NULL if the index exceeds the length of the array - and `spark.sql.ansi.enabled` is set to false. If `spark.sql.ansi.enabled` is set to true, - it throws ArrayIndexOutOfBoundsException for invalid indices. + Round the given value to `scale` decimal places using HALF_EVEN rounding mode if `scale` >= 0 + or at integral part when `scale` < 0. - .. versionadded:: 3.5.0 + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - inputs : :class:`~pyspark.sql.Column` or str - Input columns or strings. + col : :class:`~pyspark.sql.Column` or column name + The target column or column name to compute the round on. + A column that evaluates to a numeric. + scale : :class:`~pyspark.sql.Column` or int, optional + An optional parameter to control the rounding behavior. + A column that evaluates to an integer. Must be a constant. + + .. versionchanged:: 4.0.0 + Support Column type. + + Returns + ------- + :class:`~pyspark.sql.Column` + A column for the rounded value. + Returns a column of the same type as the input. Examples -------- - >>> df = spark.createDataFrame([(1, "scala", "java")], ['a', 'b', 'c']) - >>> df.select(elt(df.a, df.b, df.c).alias('r')).collect() - [Row(r='scala')] - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + Example 1: Compute the rounded of a column value - sc = _get_active_spark_context() - return _invoke_function("elt", _to_seq(sc, inputs, _to_java_column)) + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.bround(sf.lit(2.5))).show() + +--------------+ + |bround(2.5, 0)| + +--------------+ + | 2.0| + +--------------+ + Example 2: Compute the rounded of a column value with a specified scale -@_try_remote_functions -def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.bround(sf.lit(2.1267), sf.lit(2))).show() + +-----------------+ + |bround(2.1267, 2)| + +-----------------+ + | 2.13| + +-----------------+ """ - Returns the index (1-based) of the given string (`str`) in the comma-delimited - list (`strArray`). Returns 0, if the string was not found or if the given string (`str`) - contains a comma. + if scale is None: + return _invoke_function_over_columns("bround", col) + else: + scale = _enum_to_value(scale) + scale = lit(scale) if isinstance(scale, int) else scale + return _invoke_function_over_columns("bround", col, scale) - .. versionadded:: 3.5.0 - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - The given string to be found. - A column that evaluates to a string. - str_array : :class:`~pyspark.sql.Column` or str - The comma-delimited list. - A column that evaluates to a string. +@_try_remote_functions +def shiftLeft(col: "ColumnOrName", numBits: int) -> Column: + """Shift the given value numBits left. - Examples - -------- - >>> df = spark.createDataFrame([("ab", "abc,b,ab,c,def")], ['a', 'b']) - >>> df.select(find_in_set(df.a, df.b).alias('r')).collect() - [Row(r=3)] + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 3.2.0 + Use :func:`shiftleft` instead. """ - return _invoke_function_over_columns("find_in_set", str, str_array) + warnings.warn("Deprecated in 3.2, use shiftleft instead.", FutureWarning) + return shiftleft(col, numBits) @_try_remote_functions -def lcase(str: "ColumnOrName") -> Column: - """ - Returns `str` with all characters changed to lowercase. +def shiftleft(col: "ColumnOrName", numBits: int) -> Column: + """Shift the given value numBits left. - .. versionadded:: 3.5.0 + .. versionadded:: 3.2.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + input column of values to shift. + A column that evaluates to an integer or long. + numBits : int + number of bits to shift. + A column that evaluates to an integer. - See Also - -------- - :meth:`pyspark.sql.functions.lower` - :meth:`pyspark.sql.functions.ucase` - :meth:`pyspark.sql.functions.upper` + Returns + ------- + :class:`~pyspark.sql.Column` + shifted value. + Returns a column of the same type as the input. Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.lcase(sf.lit("Spark"))).show() - +------------+ - |lcase(Spark)| - +------------+ - | spark| - +------------+ + >>> spark.range(4).select("*", sf.shiftleft('id', 1)).show() + +---+----------------+ + | id|shiftleft(id, 1)| + +---+----------------+ + | 0| 0| + | 1| 2| + | 2| 4| + | 3| 6| + +---+----------------+ """ - return _invoke_function_over_columns("lcase", str) + from pyspark.sql.classic.column import _to_java_column + return _invoke_function("shiftleft", _to_java_column(col), _enum_to_value(numBits)) -@_try_remote_functions -def ucase(str: "ColumnOrName") -> Column: - """ - Returns `str` with all characters changed to uppercase. - .. versionadded:: 3.5.0 +@_try_remote_functions +def shiftRight(col: "ColumnOrName", numBits: int) -> Column: + """(Signed) shift the given value numBits right. - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. + .. versionadded:: 1.5.0 - See Also - -------- - :meth:`pyspark.sql.functions.upper` - :meth:`pyspark.sql.functions.lcase` - :meth:`pyspark.sql.functions.lower` + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.ucase(sf.lit("Spark"))).show() - +------------+ - |ucase(Spark)| - +------------+ - | SPARK| - +------------+ + .. deprecated:: 3.2.0 + Use :func:`shiftright` instead. """ - return _invoke_function_over_columns("ucase", str) + warnings.warn("Deprecated in 3.2, use shiftright instead.", FutureWarning) + return shiftright(col, numBits) @_try_remote_functions -def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: - """ - Returns the leftmost `len`(`len` can be string type) characters from the string `str`, - if `len` is less or equal than 0 the result is an empty string. +def shiftright(col: "ColumnOrName", numBits: int) -> Column: + """(Signed) shift the given value numBits right. - .. versionadded:: 3.5.0 + .. versionadded:: 3.2.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string or binary. - len : :class:`~pyspark.sql.Column` or str - Input column or strings, the leftmost `len`. + col : :class:`~pyspark.sql.Column` or column name + input column of values to shift. + A column that evaluates to an integer or long. + numBits : int + number of bits to shift. A column that evaluates to an integer. + Returns + ------- + :class:`~pyspark.sql.Column` + shifted values. + Returns a column of the same type as the input. + Examples -------- - >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) - >>> df.select(left(df.a, df.b).alias('r')).collect() - [Row(r='Spa')] + >>> import pyspark.sql.functions as sf + >>> spark.range(4).select("*", sf.shiftright('id', 1)).show() + +---+-----------------+ + | id|shiftright(id, 1)| + +---+-----------------+ + | 0| 0| + | 1| 0| + | 2| 1| + | 3| 1| + +---+-----------------+ """ - return _invoke_function_over_columns("left", str, len) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("shiftright", _to_java_column(col), _enum_to_value(numBits)) @_try_remote_functions -def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: - """ - Returns the rightmost `len`(`len` can be string type) characters from the string `str`, - if `len` is less or equal than 0 the result is an empty string. +def shiftRightUnsigned(col: "ColumnOrName", numBits: int) -> Column: + """Unsigned shift the given value numBits right. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 - Parameters - ---------- - str : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - len : :class:`~pyspark.sql.Column` or str - Input column or strings, the rightmost `len`. - A column that evaluates to an integer. + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Examples - -------- - >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) - >>> df.select(right(df.a, df.b).alias('r')).collect() - [Row(r='SQL')] + .. deprecated:: 3.2.0 + Use :func:`shiftrightunsigned` instead. """ - return _invoke_function_over_columns("right", str, len) + warnings.warn("Deprecated in 3.2, use shiftrightunsigned instead.", FutureWarning) + return shiftrightunsigned(col, numBits) @_try_remote_functions -def mask( - col: "ColumnOrName", - upperChar: Optional["ColumnOrName"] = None, - lowerChar: Optional["ColumnOrName"] = None, - digitChar: Optional["ColumnOrName"] = None, - otherChar: Optional["ColumnOrName"] = None, -) -> Column: - """ - Masks the given string value. This can be useful for creating copies of tables with sensitive - information removed. +def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: + """Unsigned shift the given value numBits right. - .. versionadded:: 3.5.0 + .. versionadded:: 3.2.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col: :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to a string. - upperChar: :class:`~pyspark.sql.Column` or str, optional - character to replace upper-case characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. - lowerChar: :class:`~pyspark.sql.Column` or str, optional - character to replace lower-case characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. - digitChar: :class:`~pyspark.sql.Column` or str, optional - character to replace digit characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. - otherChar: :class:`~pyspark.sql.Column` or str, optional - character to replace all other characters with. Specify NULL to retain original character. - A column that evaluates to a string. Must be a constant. + col : :class:`~pyspark.sql.Column` or column name + input column of values to shift. + A column that evaluates to an integer or long. + numBits : int + number of bits to shift. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - Returns a column that evaluates to a string. + shifted value. + Returns a column of the same type as the input. Examples -------- - >>> df = spark.createDataFrame([("AbCD123-@$#",), ("abcd-EFGH-8765-4321",)], ['data']) - >>> df.select(mask(df.data).alias('r')).collect() - [Row(r='XxXXnnn-@$#'), Row(r='xxxx-XXXX-nnnn-nnnn')] - >>> df.select(mask(df.data, lit('Y')).alias('r')).collect() - [Row(r='YxYYnnn-@$#'), Row(r='xxxx-YYYY-nnnn-nnnn')] - >>> df.select(mask(df.data, lit('Y'), lit('y')).alias('r')).collect() - [Row(r='YyYYnnn-@$#'), Row(r='yyyy-YYYY-nnnn-nnnn')] - >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d')).alias('r')).collect() - [Row(r='YyYYddd-@$#'), Row(r='yyyy-YYYY-dddd-dddd')] - >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d'), lit('*')).alias('r')).collect() - [Row(r='YyYYddd****'), Row(r='yyyy*YYYY*dddd*dddd')] + >>> import pyspark.sql.functions as sf + >>> spark.range(4).select("*", sf.shiftrightunsigned(sf.col('id') - 2, 1)).show() + +---+-------------------------------+ + | id|shiftrightunsigned((id - 2), 1)| + +---+-------------------------------+ + | 0| 9223372036854775807| + | 1| 9223372036854775807| + | 2| 0| + | 3| 0| + +---+-------------------------------+ """ + from pyspark.sql.classic.column import _to_java_column - _upperChar = lit("X") if upperChar is None else upperChar - _lowerChar = lit("x") if lowerChar is None else lowerChar - _digitChar = lit("n") if digitChar is None else digitChar - _otherChar = lit(None) if otherChar is None else otherChar - return _invoke_function_over_columns( - "mask", col, _upperChar, _lowerChar, _digitChar, _otherChar - ) + return _invoke_function("shiftrightunsigned", _to_java_column(col), _enum_to_value(numBits)) @_try_remote_functions -def collate(col: "ColumnOrName", collation: str) -> Column: - """ - Marks a given column with specified collation. +def spark_partition_id() -> Column: + """A column for partition ID. - .. versionadded:: 4.0.0 + .. versionadded:: 1.6.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - Target string column to work on. - collation : str - Target collation name. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column of string type, where each value has the specified collation. - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("collate", _to_java_column(col), _enum_to_value(collation)) - - -@_try_remote_functions -def collation(col: "ColumnOrName") -> Column: - """ - Returns the collation name of a given column. - - .. versionadded:: 4.0.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - Target string column to work on. - A column that evaluates to a string. + Notes + ----- + This is non deterministic because it depends on data partitioning and task scheduling. Returns ------- :class:`~pyspark.sql.Column` - collation name of a given expression. - Returns a column that evaluates to a string. + partition id the record belongs to. Examples -------- - >>> df = spark.createDataFrame([('name',)], ['dt']) - >>> df.select(collation('dt').alias('collation')).show(truncate=False) - +--------------------------+ - |collation | - +--------------------------+ - |SYSTEM.BUILTIN.UTF8_BINARY| - +--------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(10, numPartitions=5).select("*", sf.spark_partition_id()).show() + +---+--------------------+ + | id|SPARK_PARTITION_ID()| + +---+--------------------+ + | 0| 0| + | 1| 0| + | 2| 1| + | 3| 1| + | 4| 2| + | 5| 2| + | 6| 3| + | 7| 3| + | 8| 4| + | 9| 4| + +---+--------------------+ """ - return _invoke_function_over_columns("collation", col) + return _invoke_function("spark_partition_id") @_try_remote_functions -def quote(col: "ColumnOrName") -> Column: - r"""Returns `str` enclosed by single quotes and each instance of - single quote in it is preceded by a backslash. +def expr(str: str) -> Column: + """Parses the expression string into the column that it represents - .. versionadded:: 4.1.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to be quoted. - A column that evaluates to a string. + str : expression string + expression defined in string. Returns ------- :class:`~pyspark.sql.Column` - quoted string - Returns a column that evaluates to a string. + column representing the expression. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame(["Don't"], "STRING") - >>> df.select("*", sf.quote("value")).show() + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([["Alice"], ["Bob"]], ["name"]) + >>> df.select("*", sf.expr("length(name)")).show() +-----+------------+ - |value|quote(value)| + | name|length(name)| +-----+------------+ - |Don't| 'Don\'t'| + |Alice| 5| + | Bob| 3| +-----+------------+ """ - return _invoke_function_over_columns("quote", col) - - -# ---------------------- Bitwise Functions ---------------------- - + return _invoke_function("expr", str) -@_try_remote_functions -def bitwiseNOT(col: "ColumnOrName") -> Column: - """ - Computes bitwise not. - .. versionadded:: 1.4.0 +@overload +def struct(*cols: "ColumnOrName") -> Column: ... - .. versionchanged:: 3.4.0 - Supports Spark Connect. - .. deprecated:: 3.2.0 - Use :func:`bitwise_not` instead. - """ - warnings.warn("Deprecated in 3.2, use bitwise_not instead.", FutureWarning) - return bitwise_not(col) +@overload +def struct(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... @_try_remote_functions -def bitwise_not(col: "ColumnOrName") -> Column: - """ - Computes bitwise not. +def struct( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], +) -> Column: + """Creates a new struct column. - .. versionadded:: 3.2.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" - ... ).select("*", sf.bitwise_not("value")).show() - +-----+------+ - |value|~value| - +-----+------+ - | 0| -1| - | 1| -2| - | 2| -3| - | 3| -4| - | NULL| NULL| - +-----+------+ - """ - return _invoke_function_over_columns("bitwise_not", col) - - -@_try_remote_functions -def bit_count(col: "ColumnOrName") -> Column: - """ - Returns the number of bits that are set in the argument expr as an unsigned 64-bit integer, - or NULL if the argument is NULL. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral or boolean. + cols : list, set, :class:`~pyspark.sql.Column` or column name + column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct. + Each a column of any type. Returns ------- :class:`~pyspark.sql.Column` - the number of bits that are set in the argument expr as an unsigned 64-bit integer, - or NULL if the argument is NULL. - Returns a column that evaluates to an integer. + a struct type column of given columns. + Returns a column that evaluates to a struct. See Also -------- - :meth:`pyspark.sql.functions.bit_get` + :meth:`pyspark.sql.functions.named_struct` Examples -------- - >>> from pyspark.sql import functions as sf - >>> spark.sql( - ... "SELECT * FROM VALUES (0), (1), (2), (3), (NULL) AS TAB(value)" - ... ).select("*", sf.bit_count("value")).show() - +-----+----------------+ - |value|bit_count(value)| - +-----+----------------+ - | 0| 0| - | 1| 1| - | 2| 1| - | 3| 2| - | NULL| NULL| - +-----+----------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) + >>> df.select("*", sf.struct('age', df.name)).show() + +-----+---+-----------------+ + | name|age|struct(age, name)| + +-----+---+-----------------+ + |Alice| 2| {2, Alice}| + | Bob| 5| {5, Bob}| + +-----+---+-----------------+ """ - return _invoke_function_over_columns("bit_count", col) + if len(cols) == 1 and isinstance(cols[0], (list, set)): + cols = cols[0] # type: ignore[assignment] + return _invoke_function_over_seq_of_columns("struct", cols) # type: ignore[arg-type] @_try_remote_functions -def bit_get(col: "ColumnOrName", pos: "ColumnOrName") -> Column: +def named_struct(*cols: "ColumnOrName") -> Column: """ - Returns the value of the bit (0 or 1) at the specified position. - The positions are numbered from right to left, starting at zero. - The position argument cannot be negative. + Creates a struct with the given field names and values. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. - pos : :class:`~pyspark.sql.Column` or column name - The positions are numbered from right to left, starting at zero. - A column that evaluates to an integer. + cols : :class:`~pyspark.sql.Column` or column name + list of columns to work on. Returns ------- :class:`~pyspark.sql.Column` - the value of the bit (0 or 1) at the specified position. - Returns a column that evaluates to a byte. See Also -------- - :meth:`pyspark.sql.functions.bit_count` - :meth:`pyspark.sql.functions.getbit` + :meth:`pyspark.sql.functions.struct` Examples -------- - Example 1: Get the bit with a literal position - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[2],[3],[None]], ["value"]) - >>> df.select("*", sf.bit_get("value", sf.lit(1))).show() - +-----+-----------------+ - |value|bit_get(value, 1)| - +-----+-----------------+ - | 1| 0| - | 2| 1| - | 3| 1| - | NULL| NULL| - +-----+-----------------+ - - Example 2: Get the bit with a column position - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) - >>> df.select("*", sf.bit_get(df.value, "pos")).show() - +-----+----+-------------------+ - |value| pos|bit_get(value, pos)| - +-----+----+-------------------+ - | 1| 2| 0| - | 2| 1| 1| - | 3|NULL| NULL| - | NULL| 1| NULL| - +-----+----+-------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, 2)], ['a', 'b']) + >>> df.select("*", sf.named_struct(sf.lit('x'), df.a, sf.lit('y'), "b")).show() + +---+---+------------------------+ + | a| b|named_struct(x, a, y, b)| + +---+---+------------------------+ + | 1| 2| {1, 2}| + +---+---+------------------------+ """ - return _invoke_function_over_columns("bit_get", col, pos) + return _invoke_function_over_seq_of_columns("named_struct", cols) @_try_remote_functions -def getbit(col: "ColumnOrName", pos: "ColumnOrName") -> Column: +def greatest(*cols: "ColumnOrName") -> Column: """ - Returns the value of the bit (0 or 1) at the specified position. - The positions are numbered from right to left, starting at zero. - The position argument cannot be negative. + Returns the greatest value of the list of column names, skipping null values. + This function takes at least 2 parameters. It will return null if all parameters are null. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. - pos : :class:`~pyspark.sql.Column` or column name - The positions are numbered from right to left, starting at zero. - A column that evaluates to an integer. + cols: :class:`~pyspark.sql.Column` or column name + columns to check for greatest value. + Each a column of any orderable type. Returns ------- :class:`~pyspark.sql.Column` - the value of the bit (0 or 1) at the specified position. - Returns a column that evaluates to a byte. + greatest value. + Returns a column of the same type as the input. See Also -------- - :meth:`pyspark.sql.functions.bit_get` - :meth:`pyspark.sql.functions.bit_count` + :meth:`pyspark.sql.functions.least` Examples -------- - Example 1: Get the bit with a literal position - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[1], [2], [3], [None]], ["value"] - ... ).select("*", sf.getbit("value", sf.lit(1))).show() - +-----+----------------+ - |value|getbit(value, 1)| - +-----+----------------+ - | 1| 0| - | 2| 1| - | 3| 1| - | NULL| NULL| - +-----+----------------+ - - Example 2: Get the bit with a column position - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1,2],[2,1],[3,None],[None,1]], ["value", "pos"]) - >>> df.select("*", sf.getbit(df.value, "pos")).show() - +-----+----+------------------+ - |value| pos|getbit(value, pos)| - +-----+----+------------------+ - | 1| 2| 0| - | 2| 1| 1| - | 3|NULL| NULL| - | NULL| 1| NULL| - +-----+----+------------------+ + >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) + >>> df.select("*", sf.greatest(df.a, "b", df.c)).show() + +---+---+---+-----------------+ + | a| b| c|greatest(a, b, c)| + +---+---+---+-----------------+ + | 1| 4| 3| 4| + +---+---+---+-----------------+ """ - return _invoke_function_over_columns("getbit", col, pos) + if len(cols) < 2: + raise PySparkValueError( + errorClass="WRONG_NUM_COLUMNS", + messageParameters={"func_name": "greatest", "num_cols": "2"}, + ) + return _invoke_function_over_seq_of_columns("greatest", cols) @_try_remote_functions -def shiftLeft(col: "ColumnOrName", numBits: int) -> Column: - """Shift the given value numBits left. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. deprecated:: 3.2.0 - Use :func:`shiftleft` instead. +def least(*cols: "ColumnOrName") -> Column: """ - warnings.warn("Deprecated in 3.2, use shiftleft instead.", FutureWarning) - return shiftleft(col, numBits) - - -@_try_remote_functions -def shiftleft(col: "ColumnOrName", numBits: int) -> Column: - """Shift the given value numBits left. + Returns the least value of the list of column names, skipping null values. + This function takes at least 2 parameters. It will return null if all parameters are null. - .. versionadded:: 3.2.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to shift. - A column that evaluates to an integer or long. - numBits : int - number of bits to shift. - A column that evaluates to an integer. + cols : :class:`~pyspark.sql.Column` or column name + column names or columns to be compared + Each a column of any orderable type. Returns ------- :class:`~pyspark.sql.Column` - shifted value. + least value. Returns a column of the same type as the input. + See Also + -------- + :meth:`pyspark.sql.functions.greatest` + Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(4).select("*", sf.shiftleft('id', 1)).show() - +---+----------------+ - | id|shiftleft(id, 1)| - +---+----------------+ - | 0| 0| - | 1| 2| - | 2| 4| - | 3| 6| - +---+----------------+ - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("shiftleft", _to_java_column(col), _enum_to_value(numBits)) - - -@_try_remote_functions -def shiftRight(col: "ColumnOrName", numBits: int) -> Column: - """(Signed) shift the given value numBits right. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. deprecated:: 3.2.0 - Use :func:`shiftright` instead. + >>> df = spark.createDataFrame([(1, 4, 3)], ['a', 'b', 'c']) + >>> df.select("*", sf.least(df.a, "b", df.c)).show() + +---+---+---+--------------+ + | a| b| c|least(a, b, c)| + +---+---+---+--------------+ + | 1| 4| 3| 1| + +---+---+---+--------------+ """ - warnings.warn("Deprecated in 3.2, use shiftright instead.", FutureWarning) - return shiftright(col, numBits) + if len(cols) < 2: + raise PySparkValueError( + errorClass="WRONG_NUM_COLUMNS", + messageParameters={"func_name": "least", "num_cols": "2"}, + ) + return _invoke_function_over_seq_of_columns("least", cols) @_try_remote_functions -def shiftright(col: "ColumnOrName", numBits: int) -> Column: - """(Signed) shift the given value numBits right. +def when(condition: Column, value: Any) -> Column: + """Evaluates a list of conditions and returns one of multiple possible result expressions. + If :func:`pyspark.sql.Column.otherwise` is not invoked, None is returned for unmatched + conditions. - .. versionadded:: 3.2.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to shift. - A column that evaluates to an integer or long. - numBits : int - number of bits to shift. - A column that evaluates to an integer. + condition : :class:`~pyspark.sql.Column` + a boolean :class:`~pyspark.sql.Column` expression. + A column that evaluates to a boolean. + value : + a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of any type. Returns ------- :class:`~pyspark.sql.Column` - shifted values. + column representing when expression. Returns a column of the same type as the input. + See Also + -------- + :meth:`pyspark.sql.Column.when` + :meth:`pyspark.sql.Column.otherwise` + Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(4).select("*", sf.shiftright('id', 1)).show() - +---+-----------------+ - | id|shiftright(id, 1)| - +---+-----------------+ - | 0| 0| - | 1| 0| - | 2| 1| - | 3| 1| - +---+-----------------+ - """ - from pyspark.sql.classic.column import _to_java_column + >>> df = spark.range(3) + >>> df.select("*", sf.when(df['id'] == 2, 3).otherwise(4)).show() + +---+------------------------------------+ + | id|CASE WHEN (id = 2) THEN 3 ELSE 4 END| + +---+------------------------------------+ + | 0| 4| + | 1| 4| + | 2| 3| + +---+------------------------------------+ - return _invoke_function("shiftright", _to_java_column(col), _enum_to_value(numBits)) + >>> df.select("*", sf.when(df.id == 2, df.id + 1)).show() + +---+------------------------------------+ + | id|CASE WHEN (id = 2) THEN (id + 1) END| + +---+------------------------------------+ + | 0| NULL| + | 1| NULL| + | 2| 3| + +---+------------------------------------+ + """ + # Explicitly not using ColumnOrName type here to make reading condition less opaque + if not isinstance(condition, Column): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column", + "arg_name": "condition", + "arg_type": type(condition).__name__, + }, + ) + value = _enum_to_value(value) + v = value._jc if isinstance(value, Column) else _enum_to_value(value) + return _invoke_function("when", condition._jc, v) -@_try_remote_functions -def shiftRightUnsigned(col: "ColumnOrName", numBits: int) -> Column: - """Unsigned shift the given value numBits right. - .. versionadded:: 1.5.0 +@overload +def log(arg1: "ColumnOrName") -> Column: ... - .. versionchanged:: 3.4.0 - Supports Spark Connect. - .. deprecated:: 3.2.0 - Use :func:`shiftrightunsigned` instead. - """ - warnings.warn("Deprecated in 3.2, use shiftrightunsigned instead.", FutureWarning) - return shiftrightunsigned(col, numBits) +@overload +def log(arg1: float, arg2: "ColumnOrName") -> Column: ... @_try_remote_functions -def shiftrightunsigned(col: "ColumnOrName", numBits: int) -> Column: - """Unsigned shift the given value numBits right. +def log(arg1: Union["ColumnOrName", float], arg2: Optional["ColumnOrName"] = None) -> Column: + """Returns the first argument-based logarithm of the second argument. - .. versionadded:: 3.2.0 + If there is only one argument, then this takes the natural logarithm of the argument. + + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to shift. - A column that evaluates to an integer or long. - numBits : int - number of bits to shift. - A column that evaluates to an integer. + arg1 : :class:`~pyspark.sql.Column`, str or float + base number or actual number (in this case base is `e`). + A column that evaluates to a double. + arg2 : :class:`~pyspark.sql.Column`, str or float, optional + number to calculate logariphm for. + A column that evaluates to a double. Returns ------- :class:`~pyspark.sql.Column` - shifted value. - Returns a column of the same type as the input. + logariphm of given value. + Returns a column that evaluates to a double. + + See Also + -------- + :meth:`pyspark.sql.functions.ln` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(4).select("*", sf.shiftrightunsigned(sf.col('id') - 2, 1)).show() - +---+-------------------------------+ - | id|shiftrightunsigned((id - 2), 1)| - +---+-------------------------------+ - | 0| 9223372036854775807| - | 1| 9223372036854775807| - | 2| 0| - | 3| 0| - +---+-------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + Example 1: Specify both base number and the input value - return _invoke_function("shiftrightunsigned", _to_java_column(col), _enum_to_value(numBits)) + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") + >>> df.select("*", sf.log(2.0, df.value)).show() + +-----+---------------+ + |value|LOG(2.0, value)| + +-----+---------------+ + | 1| 0.0| + | 2| 1.0| + | 4| 2.0| + +-----+---------------+ + + Example 2: Return NULL for invalid input values + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (0), (-1), (NULL) AS t(value)") + >>> df.select("*", sf.log(3.0, df.value)).show() + +-----+------------------+ + |value| LOG(3.0, value)| + +-----+------------------+ + | 1| 0.0| + | 2|0.6309297535714...| + | 0| NULL| + | -1| NULL| + | NULL| NULL| + +-----+------------------+ + + Example 3: Specify only the input value (Natural logarithm) + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT * FROM VALUES (1), (2), (4) AS t(value)") + >>> df.select("*", sf.log(df.value)).show() + +-----+------------------+ + |value| ln(value)| + +-----+------------------+ + | 1| 0.0| + | 2|0.6931471805599...| + | 4|1.3862943611198...| + +-----+------------------+ + """ + from pyspark.sql.classic.column import _to_java_column -# ---------------------- Date and Timestamp Functions ---------------------- + if arg2 is None: + return _invoke_function_over_columns("log", cast("ColumnOrName", arg1)) + else: + return _invoke_function("log", _enum_to_value(arg1), _to_java_column(arg2)) @_try_remote_functions -def curdate() -> Column: - """ - Returns the current date at the start of query evaluation as a :class:`DateType` column. - All calls of current_date within the same query return the same value. +def ln(col: "ColumnOrName") -> Column: + """Returns the natural logarithm of the argument. .. versionadded:: 3.5.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column to calculate logariphm for. + A column that evaluates to a double. + Returns ------- :class:`~pyspark.sql.Column` - current date. + natural logarithm of given value. + Returns a column that evaluates to a double. See Also -------- - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.current_timestamp` - :meth:`pyspark.sql.functions.localtimestamp` + :meth:`pyspark.sql.functions.log` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.curdate()).show() # doctest: +SKIP - +--------------+ - |current_date()| - +--------------+ - | 2022-08-26| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> spark.range(10).select("*", sf.ln('id')).show() + +---+------------------+ + | id| ln(id)| + +---+------------------+ + | 0| NULL| + | 1| 0.0| + | 2|0.6931471805599...| + | 3|1.0986122886681...| + | 4|1.3862943611198...| + | 5|1.6094379124341...| + | 6| 1.791759469228...| + | 7|1.9459101490553...| + | 8|2.0794415416798...| + | 9|2.1972245773362...| + +---+------------------+ """ - return _invoke_function("curdate") + return _invoke_function_over_columns("ln", col) @_try_remote_functions -def current_date() -> Column: - """ - Returns the current date at the start of query evaluation as a :class:`DateType` column. - All calls of current_date within the same query return the same value. +def log2(col: "ColumnOrName") -> Column: + """Returns the base-2 logarithm of the argument. .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column to calculate logariphm for. + A column that evaluates to a double. + Returns ------- :class:`~pyspark.sql.Column` - current date. - - See Also - -------- - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.curdate` - :meth:`pyspark.sql.functions.current_timestamp` - :meth:`pyspark.sql.functions.localtimestamp` + logariphm of given value. + Returns a column that evaluates to a double. Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.current_date()).show() # doctest: +SKIP - +--------------+ - |current_date()| - +--------------+ - | 2022-08-26| - +--------------+ + >>> spark.range(10).select("*", sf.log2('id')).show() + +---+------------------+ + | id| LOG2(id)| + +---+------------------+ + | 0| NULL| + | 1| 0.0| + | 2| 1.0| + | 3| 1.584962500721...| + | 4| 2.0| + | 5| 2.321928094887...| + | 6| 2.584962500721...| + | 7| 2.807354922057...| + | 8| 3.0| + | 9|3.1699250014423...| + +---+------------------+ """ - return _invoke_function("current_date") + return _invoke_function_over_columns("log2", col) @_try_remote_functions -def current_timezone() -> Column: +def conv(col: "ColumnOrName", fromBase: int, toBase: int) -> Column: """ - Returns the current session local timezone. + Convert a number in a string column from one base to another. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column to convert base for. + A column that evaluates to a string. + fromBase: int + from base number. + A column that evaluates to an integer. + toBase: int + to base number. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - current session local timezone. - - See Also - -------- - :meth:`pyspark.sql.functions.convert_timezone` + logariphm of given value. + Returns a column that evaluates to a string. Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.current_timezone()).show() - +-------------------+ - | current_timezone()| - +-------------------+ - |America/Los_Angeles| - +-------------------+ - - Switch the timezone to Shanghai. - - >>> spark.conf.set("spark.sql.session.timeZone", "Asia/Shanghai") - >>> spark.range(1).select(sf.current_timezone()).show() - +------------------+ - |current_timezone()| - +------------------+ - | Asia/Shanghai| - +------------------+ - - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> df = spark.createDataFrame([("010101",), ( "101",), ("001",)], ['n']) + >>> df.select("*", sf.conv(df.n, 2, 16)).show() + +------+--------------+ + | n|conv(n, 2, 16)| + +------+--------------+ + |010101| 15| + | 101| 5| + | 001| 1| + +------+--------------+ """ - return _invoke_function("current_timezone") - - -@overload -def current_time() -> Column: ... - + from pyspark.sql.classic.column import _to_java_column -@overload -def current_time(precision: int) -> Column: ... + return _invoke_function( + "conv", _to_java_column(col), _enum_to_value(fromBase), _enum_to_value(toBase) + ) @_try_remote_functions -def current_time(precision: Optional[int] = None) -> Column: +def factorial(col: "ColumnOrName") -> Column: """ - Returns the current time at the start of query evaluation as a :class:`TimeType` column. All - calls of current_time within the same query return the same value. + Computes the factorial of the given value. - .. versionadded:: 4.1.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - precision: literal int, optional - number in the range [0..6], indicating how many fractional digits of seconds to include. - If omitted, the default is 6. + col : :class:`~pyspark.sql.Column` or str + a column to calculate factorial for. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - current time. - - See Also - -------- - :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.current_timestamp` + factorial of given value. + Returns a column that evaluates to a long. Examples -------- - Example 1: Current time with default precision - >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.current_time().alias("time")).show() # doctest: +SKIP - +---------------+ - | time| - +---------------+ - |16:57:04.304361| - +---------------+ + >>> spark.range(10).select("*", sf.factorial('id')).show() + +---+-------------+ + | id|factorial(id)| + +---+-------------+ + | 0| 1| + | 1| 1| + | 2| 2| + | 3| 6| + | 4| 24| + | 5| 120| + | 6| 720| + | 7| 5040| + | 8| 40320| + | 9| 362880| + +---+-------------+ + """ + return _invoke_function_over_columns("factorial", col) - Example 2: Current time with specified precision - >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.current_time(3).alias("time")).show() # doctest: +SKIP - +------------+ - | time| - +------------+ - |16:57:04.304| - +------------+ - """ - if precision is None: - return _invoke_function("current_time") - else: - return _invoke_function("current_time", _enum_to_value(precision)) +# --------------- Window functions ------------------------ @_try_remote_functions -def current_timestamp() -> Column: +def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: """ - Returns the current timestamp at the start of query evaluation as a :class:`TimestampType` - column. All calls of current_timestamp within the same query return the same value. + Window function: returns the value that is `offset` rows before the current row, and + `default` if there is less than `offset` rows before the current row. For example, + an `offset` of one will return the previous row at any given point in the window partition. - .. versionadded:: 1.5.0 + This is equivalent to the LAG function in SQL. + + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + name of column or expression + offset : int, optional default 1 + number of row to extend + default : optional + default value + Returns ------- :class:`~pyspark.sql.Column` - current date and time. + value before current row based on `offset`. See Also -------- - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.curdate` - :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.localtimestamp` + :meth:`pyspark.sql.functions.lead` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.current_timestamp()).show(truncate=False) # doctest: +SKIP - +-----------------------+ - |current_timestamp() | - +-----------------------+ - |2022-08-26 21:23:22.716| - +-----------------------+ - """ - return _invoke_function("current_timestamp") - - -@_try_remote_functions -def now() -> Column: - """ - Returns the current timestamp at the start of query evaluation. - - .. versionadded:: 3.5.0 + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ - Returns - ------- - :class:`~pyspark.sql.Column` - current timestamp at the start of query evaluation. + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("previous_value", sf.lag("c2").over(w)).show() + +---+---+--------------+ + | c1| c2|previous_value| + +---+---+--------------+ + | a| 1| NULL| + | a| 2| 1| + | a| 3| 2| + | b| 2| NULL| + | b| 8| 2| + +---+---+--------------+ - See Also - -------- - :meth:`pyspark.sql.functions.curdate` - :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.current_timestamp` - :meth:`pyspark.sql.functions.localtimestamp` + >>> df.withColumn("previous_value", sf.lag("c2", 1, 0).over(w)).show() + +---+---+--------------+ + | c1| c2|previous_value| + +---+---+--------------+ + | a| 1| 0| + | a| 2| 1| + | a| 3| 2| + | b| 2| 0| + | b| 8| 2| + +---+---+--------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.now()).show(truncate=False) # doctest: +SKIP - +--------------------------+ - |now() | - +--------------------------+ - |2023-12-08 15:18:18.482269| - +--------------------------+ + >>> df.withColumn("previous_value", sf.lag("c2", 2, -1).over(w)).show() + +---+---+--------------+ + | c1| c2|previous_value| + +---+---+--------------+ + | a| 1| -1| + | a| 2| -1| + | a| 3| 1| + | b| 2| -1| + | b| 8| -1| + +---+---+--------------+ """ - return _invoke_function("now") + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "lag", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) + ) @_try_remote_functions -def localtimestamp() -> Column: +def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: """ - Returns the current timestamp without time zone at the start of query evaluation - as a timestamp without time zone column. All calls of localtimestamp within the - same query return the same value. + Window function: returns the value that is `offset` rows after the current row, and + `default` if there is less than `offset` rows after the current row. For example, + an `offset` of one will return the next row at any given point in the window partition. - .. versionadded:: 3.4.0 + This is equivalent to the LEAD function in SQL. + + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + name of column or expression + offset : int, optional default 1 + number of row to extend + default : optional + default value + Returns ------- :class:`~pyspark.sql.Column` - current local date and time. + value after current row based on `offset`. See Also -------- - :meth:`pyspark.sql.functions.now` - :meth:`pyspark.sql.functions.curdate` - :meth:`pyspark.sql.functions.current_date` - :meth:`pyspark.sql.functions.current_timestamp` + :meth:`pyspark.sql.functions.lag` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.localtimestamp()).show(truncate=False) # doctest: +SKIP - +-----------------------+ - |localtimestamp() | - +-----------------------+ - |2022-08-26 21:28:34.639| - +-----------------------+ + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ + + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("next_value", sf.lead("c2").over(w)).show() + +---+---+----------+ + | c1| c2|next_value| + +---+---+----------+ + | a| 1| 2| + | a| 2| 3| + | a| 3| NULL| + | b| 2| 8| + | b| 8| NULL| + +---+---+----------+ + + >>> df.withColumn("next_value", sf.lead("c2", 1, 0).over(w)).show() + +---+---+----------+ + | c1| c2|next_value| + +---+---+----------+ + | a| 1| 2| + | a| 2| 3| + | a| 3| 0| + | b| 2| 8| + | b| 8| 0| + +---+---+----------+ + + >>> df.withColumn("next_value", sf.lead("c2", 2, -1).over(w)).show() + +---+---+----------+ + | c1| c2|next_value| + +---+---+----------+ + | a| 1| 3| + | a| 2| -1| + | a| 3| -1| + | b| 2| -1| + | b| 8| -1| + +---+---+----------+ """ - return _invoke_function("localtimestamp") + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "lead", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) + ) @_try_remote_functions -def date_format(date: "ColumnOrName", format: str) -> Column: +def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = False) -> Column: """ - Converts a date/timestamp/string to a value of string in the format specified by the date - format given by the second argument. + Window function: returns the value that is the `offset`\\th row of the window frame + (counting from 1), and `null` if the size of window frame is less than `offset` rows. - A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All - pattern letters of `datetime pattern`_. can be used. + It will return the `offset`\\th non-null value it sees when `ignoreNulls` is set to + true. If all values are null, then null is returned. - .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html + This is equivalent to the nth_value function in SQL. - .. versionadded:: 1.5.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Notes - ----- - Whenever possible, use specialized functions like `year`. - Parameters ---------- - date : :class:`~pyspark.sql.Column` or column name - input column of values to format. - A column that evaluates to a timestamp or time. - format: literal string - format to use to represent datetime values. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + name of column or expression + offset : int + number of row to use as the value + ignoreNulls : bool, optional + indicates the Nth value should skip null in the + determination of which row to use + + Returns + ------- + :class:`~pyspark.sql.Column` + value of nth row. See Also -------- - :meth:`pyspark.sql.functions.to_date` - :meth:`pyspark.sql.functions.to_timestamp` - :meth:`pyspark.sql.functions.to_timestamp_ltz` - :meth:`pyspark.sql.functions.to_timestamp_ntz` - :meth:`pyspark.sql.functions.to_utc_timestamp` - :meth:`pyspark.sql.functions.try_to_timestamp` - - Returns - ------- - :class:`~pyspark.sql.Column` - string value representing formatted datetime. - Returns a column that evaluates to a string. + :meth:`pyspark.sql.functions.first_value` + :meth:`pyspark.sql.functions.last_value` Examples -------- - Example 1: Format a string column representing dates - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.date_format('dt', 'MM/dd/yyyy')).show() - +----------+----------+---------------------------+ - | dt|typeof(dt)|date_format(dt, MM/dd/yyyy)| - +----------+----------+---------------------------+ - |2015-04-08| string| 04/08/2015| - |2024-10-31| string| 10/31/2024| - +----------+----------+---------------------------+ - - Example 2: Format a string column representing timestamp - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.date_format('ts', 'yy=MM=dd HH=mm=ss')).show() - +-------------------+----------+----------------------------------+ - | ts|typeof(ts)|date_format(ts, yy=MM=dd HH=mm=ss)| - +-------------------+----------+----------------------------------+ - |2015-04-08 13:08:15| string| 15=04=08 13=08=15| - |2024-10-31 10:09:16| string| 24=10=31 10=09=16| - +-------------------+----------+----------------------------------+ - - Example 3: Format a date column - - >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.date_format('dt', 'yy--MM--dd')).show() - +----------+----------+---------------------------+ - | dt|typeof(dt)|date_format(dt, yy--MM--dd)| - +----------+----------+---------------------------+ - |2015-04-08| date| 15--04--08| - |2024-10-31| date| 24--10--31| - +----------+----------+---------------------------+ + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ - Example 4: Format a timestamp column + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("nth_value", sf.nth_value("c2", 1).over(w)).show() + +---+---+---------+ + | c1| c2|nth_value| + +---+---+---------+ + | a| 1| 1| + | a| 2| 1| + | a| 3| 1| + | b| 2| 2| + | b| 8| 2| + +---+---+---------+ - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.date_format('ts', 'yy=MM=dd HH=mm=ss')).show() - +-------------------+----------+----------------------------------+ - | ts|typeof(ts)|date_format(ts, yy=MM=dd HH=mm=ss)| - +-------------------+----------+----------------------------------+ - |2015-04-08 13:08:15| timestamp| 15=04=08 13=08=15| - |2024-10-31 10:09:16| timestamp| 24=10=31 10=09=16| - +-------------------+----------+----------------------------------+ + >>> df.withColumn("nth_value", sf.nth_value("c2", 2).over(w)).show() + +---+---+---------+ + | c1| c2|nth_value| + +---+---+---------+ + | a| 1| NULL| + | a| 2| 2| + | a| 3| 2| + | b| 2| NULL| + | b| 8| 8| + +---+---+---------+ """ from pyspark.sql.classic.column import _to_java_column - return _invoke_function("date_format", _to_java_column(date), _enum_to_value(format)) + return _invoke_function( + "nth_value", _to_java_column(col), _enum_to_value(offset), _enum_to_value(ignoreNulls) + ) @_try_remote_functions -def year(col: "ColumnOrName") -> Column: - """ - Extract the year of a given date/timestamp as integer. - - .. versionadded:: 1.5.0 +def any_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: + """Returns some value of `col` for a group of rows. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target date/timestamp column to work on. - A column that evaluates to a date, timestamp or string. + target column to work on. + A column of any type. + ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional + if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - year part of the date/timestamp as integer. - Returns a column that evaluates to an integer. - - See Also - -------- - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.date_part` + some value of `col` for a group of rows. Examples -------- - Example 1: Extract the year from a string column representing dates - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.year('dt')).show() - +----------+----------+--------+ - | dt|typeof(dt)|year(dt)| - +----------+----------+--------+ - |2015-04-08| string| 2015| - |2024-10-31| string| 2024| - +----------+----------+--------+ - - Example 2: Extract the year from a string column representing timestamp - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.year('ts')).show() - +-------------------+----------+--------+ - | ts|typeof(ts)|year(ts)| - +-------------------+----------+--------+ - |2015-04-08 13:08:15| string| 2015| - |2024-10-31 10:09:16| string| 2024| - +-------------------+----------+--------+ - - Example 3: Extract the year from a date column - - >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.year('dt')).show() - +----------+----------+--------+ - | dt|typeof(dt)|year(dt)| - +----------+----------+--------+ - |2015-04-08| date| 2015| - |2024-10-31| date| 2024| - +----------+----------+--------+ - - Example 4: Extract the year from a timestamp column + >>> df = spark.createDataFrame( + ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.select(sf.any_value('c1'), sf.any_value('c2')).show() + +-------------+-------------+ + |any_value(c1)|any_value(c2)| + +-------------+-------------+ + | NULL| 1| + +-------------+-------------+ - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.year('ts')).show() - +-------------------+----------+--------+ - | ts|typeof(ts)|year(ts)| - +-------------------+----------+--------+ - |2015-04-08 13:08:15| timestamp| 2015| - |2024-10-31 10:09:16| timestamp| 2024| - +-------------------+----------+--------+ + >>> df.select(sf.any_value('c1', True), sf.any_value('c2', True)).show() + +-------------+-------------+ + |any_value(c1)|any_value(c2)| + +-------------+-------------+ + | a| 1| + +-------------+-------------+ """ - return _invoke_function_over_columns("year", col) + if ignoreNulls is None: + return _invoke_function_over_columns("any_value", col) + else: + ignoreNulls = _enum_to_value(ignoreNulls) + ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls + return _invoke_function_over_columns("any_value", col, ignoreNulls) @_try_remote_functions -def quarter(col: "ColumnOrName") -> Column: - """ - Extract the quarter of a given date/timestamp as integer. - - .. versionadded:: 1.5.0 +def first_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: + """Returns the first value of `col` for a group of rows. It will return the first non-null + value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target date/timestamp column to work on. - A column that evaluates to a date, timestamp or string. + target column to work on. + A column of any type. + ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional + if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - quarter of the date/timestamp as integer. - Returns a column that evaluates to an integer. + some value of `col` for a group of rows. See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.last_value` + :meth:`pyspark.sql.functions.nth_value` Examples -------- - Example 1: Extract the quarter from a string column representing dates + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] + ... ).select(sf.first_value('a'), sf.first_value('b')).show() + +--------------+--------------+ + |first_value(a)|first_value(b)| + +--------------+--------------+ + | NULL| 1| + +--------------+--------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.quarter('dt')).show() - +----------+----------+-----------+ - | dt|typeof(dt)|quarter(dt)| - +----------+----------+-----------+ - |2015-04-08| string| 2| - |2024-10-31| string| 4| - +----------+----------+-----------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] + ... ).select(sf.first_value('a', True), sf.first_value('b', True)).show() + +--------------+--------------+ + |first_value(a)|first_value(b)| + +--------------+--------------+ + | a| 1| + +--------------+--------------+ + """ + if ignoreNulls is None: + return _invoke_function_over_columns("first_value", col) + else: + ignoreNulls = _enum_to_value(ignoreNulls) + ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls + return _invoke_function_over_columns("first_value", col, ignoreNulls) - Example 2: Extract the quarter from a string column representing timestamp - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.quarter('ts')).show() - +-------------------+----------+-----------+ - | ts|typeof(ts)|quarter(ts)| - +-------------------+----------+-----------+ - |2015-04-08 13:08:15| string| 2| - |2024-10-31 10:09:16| string| 4| - +-------------------+----------+-----------+ +@_try_remote_functions +def last_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: + """Returns the last value of `col` for a group of rows. It will return the last non-null + value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. - Example 3: Extract the quarter from a date column + .. versionadded:: 3.5.0 - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.quarter('dt')).show() - +----------+----------+-----------+ - | dt|typeof(dt)|quarter(dt)| - +----------+----------+-----------+ - |2015-04-08| date| 2| - |2024-10-31| date| 4| - +----------+----------+-----------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column of any type. + ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional + if first value is null then look for first non-null value. + A column that evaluates to a boolean. Must be a constant. - Example 4: Extract the quarter from a timestamp column + Returns + ------- + :class:`~pyspark.sql.Column` + some value of `col` for a group of rows. - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.quarter('ts')).show() - +-------------------+----------+-----------+ - | ts|typeof(ts)|quarter(ts)| - +-------------------+----------+-----------+ - |2015-04-08 13:08:15| timestamp| 2| - |2024-10-31 10:09:16| timestamp| 4| - +-------------------+----------+-----------+ + See Also + -------- + :meth:`pyspark.sql.functions.first_value` + :meth:`pyspark.sql.functions.nth_value` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] + ... ).select(sf.last_value('a'), sf.last_value('b')).show() + +-------------+-------------+ + |last_value(a)|last_value(b)| + +-------------+-------------+ + | NULL| 2| + +-------------+-------------+ + + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] + ... ).select(sf.last_value('a', True), sf.last_value('b', True)).show() + +-------------+-------------+ + |last_value(a)|last_value(b)| + +-------------+-------------+ + | b| 2| + +-------------+-------------+ """ - return _invoke_function_over_columns("quarter", col) + if ignoreNulls is None: + return _invoke_function_over_columns("last_value", col) + else: + ignoreNulls = _enum_to_value(ignoreNulls) + ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls + return _invoke_function_over_columns("last_value", col, ignoreNulls) @_try_remote_functions -def month(col: "ColumnOrName") -> Column: +def count_if(col: "ColumnOrName") -> Column: """ - Extract the month of a given date/timestamp as integer. - - .. versionadded:: 1.5.0 + Aggregate function: Returns the number of `TRUE` values for the `col`. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target date/timestamp column to work on. - A column that evaluates to a date, timestamp or string. + target column to work on. + A column that evaluates to a boolean. Returns ------- :class:`~pyspark.sql.Column` - month part of the date/timestamp as integer. - Returns a column that evaluates to an integer. + the number of `TRUE` values for the `col`. See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.monthname` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.count` Examples -------- - Example 1: Extract the month from a string column representing dates + Example 1: Counting the number of even numbers in a numeric column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.month('dt')).show() - +----------+----------+---------+ - | dt|typeof(dt)|month(dt)| - +----------+----------+---------+ - |2015-04-08| string| 4| - |2024-10-31| string| 10| - +----------+----------+---------+ + >>> df = spark.createDataFrame([("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.select(sf.count_if(sf.col('c2') % 2 == 0)).show() + +------------------------+ + |count_if(((c2 % 2) = 0))| + +------------------------+ + | 3| + +------------------------+ - Example 2: Extract the month from a string column representing timestamp + Example 2: Counting the number of rows where a string column starts with a certain letter >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.month('ts')).show() - +-------------------+----------+---------+ - | ts|typeof(ts)|month(ts)| - +-------------------+----------+---------+ - |2015-04-08 13:08:15| string| 4| - |2024-10-31 10:09:16| string| 10| - +-------------------+----------+---------+ + >>> df = spark.createDataFrame( + ... [("apple",), ("banana",), ("cherry",), ("apple",), ("banana",)], ["fruit"]) + >>> df.select(sf.count_if(sf.col('fruit').startswith('a'))).show() + +------------------------------+ + |count_if(startswith(fruit, a))| + +------------------------------+ + | 2| + +------------------------------+ - Example 3: Extract the month from a date column + Example 3: Counting the number of rows where a numeric column is greater than a certain value - >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.month('dt')).show() - +----------+----------+---------+ - | dt|typeof(dt)|month(dt)| - +----------+----------+---------+ - |2015-04-08| date| 4| - |2024-10-31| date| 10| - +----------+----------+---------+ + >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,), (5,)], ["num"]) + >>> df.select(sf.count_if(sf.col('num') > 3)).show() + +-------------------+ + |count_if((num > 3))| + +-------------------+ + | 2| + +-------------------+ - Example 3: Extract the month from a timestamp column + Example 4: Counting the number of rows where a boolean column is True - >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.month('ts')).show() - +-------------------+----------+---------+ - | ts|typeof(ts)|month(ts)| - +-------------------+----------+---------+ - |2015-04-08 13:08:15| timestamp| 4| - |2024-10-31 10:09:16| timestamp| 10| - +-------------------+----------+---------+ + >>> df = spark.createDataFrame([(True,), (False,), (True,), (False,), (True,)], ["b"]) + >>> df.select(sf.count('b'), sf.count_if('b')).show() + +--------+-----------+ + |count(b)|count_if(b)| + +--------+-----------+ + | 5| 3| + +--------+-----------+ """ - return _invoke_function_over_columns("month", col) + return _invoke_function_over_columns("count_if", col) @_try_remote_functions -def dayofweek(col: "ColumnOrName") -> Column: - """ - Extract the day of the week of a given date/timestamp as integer. - Ranges from 1 for a Sunday through to 7 for a Saturday - - .. versionadded:: 2.3.0 +def histogram_numeric(col: "ColumnOrName", nBins: Column) -> Column: + """Computes a histogram on numeric 'col' using nb bins. + The return value is an array of (x,y) pairs representing the centers of the + histogram's bins. As the value of 'nb' is increased, the histogram approximation + gets finer-grained, but may yield artifacts around outliers. In practice, 20-40 + histogram bins appear to work well, with more bins being required for skewed or + smaller datasets. Note that this function creates a histogram with non-uniform + bin widths. It offers no guarantees in terms of the mean-squared-error of the + histogram, but in practice is comparable to the histograms produced by the R/S-Plus + statistical computing packages. Note: the output type of the 'x' field in the return value is + propagated from the input value consumed in the aggregate function. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target date/timestamp column to work on. - A column that evaluates to a date, timestamp or string. + target column to work on. + nBins : :class:`~pyspark.sql.Column` + number of Histogram columns. Returns ------- :class:`~pyspark.sql.Column` - day of the week for given date/timestamp as integer. - Returns a column that evaluates to an integer. - - See Also - -------- - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.weekofyear` + a histogram on numeric 'col' using nb bins. Examples -------- - Example 1: Extract the day of the week from a string column representing dates - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayofweek('dt')).show() - +----------+----------+-------------+ - | dt|typeof(dt)|dayofweek(dt)| - +----------+----------+-------------+ - |2015-04-08| string| 4| - |2024-10-31| string| 5| - +----------+----------+-------------+ - - Example 2: Extract the day of the week from a string column representing timestamp - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayofweek('ts')).show() - +-------------------+----------+-------------+ - | ts|typeof(ts)|dayofweek(ts)| - +-------------------+----------+-------------+ - |2015-04-08 13:08:15| string| 4| - |2024-10-31 10:09:16| string| 5| - +-------------------+----------+-------------+ - - Example 3: Extract the day of the week from a date column - - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayofweek('dt')).show() - +----------+----------+-------------+ - | dt|typeof(dt)|dayofweek(dt)| - +----------+----------+-------------+ - |2015-04-08| date| 4| - |2024-10-31| date| 5| - +----------+----------+-------------+ - - Example 4: Extract the day of the week from a timestamp column - - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayofweek('ts')).show() - +-------------------+----------+-------------+ - | ts|typeof(ts)|dayofweek(ts)| - +-------------------+----------+-------------+ - |2015-04-08 13:08:15| timestamp| 4| - |2024-10-31 10:09:16| timestamp| 5| - +-------------------+----------+-------------+ + >>> df = spark.range(100, numPartitions=1) + >>> df.select(sf.histogram_numeric('id', sf.lit(5))).show(truncate=False) + +-----------------------------------------------------------+ + |histogram_numeric(id, 5) | + +-----------------------------------------------------------+ + |[{11, 25.0}, {36, 24.0}, {59, 23.0}, {84, 25.0}, {98, 3.0}]| + +-----------------------------------------------------------+ """ - return _invoke_function_over_columns("dayofweek", col) + return _invoke_function_over_columns("histogram_numeric", col, nBins) @_try_remote_functions -def dayofmonth(col: "ColumnOrName") -> Column: +def ntile(n: int) -> Column: """ - Extract the day of the month of a given date/timestamp as integer. + Window function: returns the ntile group id (from 1 to `n` inclusive) + in an ordered window partition. For example, if `n` is 4, the first + quarter of the rows will get value 1, the second quarter will get 2, + the third quarter will get 3, and the last quarter will get 4. - .. versionadded:: 1.5.0 + This is equivalent to the NTILE function in SQL. + + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target date/timestamp column to work on. - A column that evaluates to a date, timestamp or string. - - See Also - -------- - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.weekofyear` + n : int + an integer Returns ------- :class:`~pyspark.sql.Column` - day of the month for given date/timestamp as integer. - Returns a column that evaluates to an integer. + portioned group id. - Examples + See Also -------- - Example 1: Extract the day of the month from a string column representing dates - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayofmonth('dt')).show() - +----------+----------+--------------+ - | dt|typeof(dt)|dayofmonth(dt)| - +----------+----------+--------------+ - |2015-04-08| string| 8| - |2024-10-31| string| 31| - +----------+----------+--------------+ - - Example 2: Extract the day of the month from a string column representing timestamp + :meth:`pyspark.sql.functions.cume_dist` + :meth:`pyspark.sql.functions.dense_rank` + :meth:`pyspark.sql.functions.percent_rank` + :meth:`pyspark.sql.functions.rank` + :meth:`pyspark.sql.functions.row_number` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayofmonth('ts')).show() - +-------------------+----------+--------------+ - | ts|typeof(ts)|dayofmonth(ts)| - +-------------------+----------+--------------+ - |2015-04-08 13:08:15| string| 8| - |2024-10-31 10:09:16| string| 31| - +-------------------+----------+--------------+ - - Example 3: Extract the day of the month from a date column + >>> from pyspark.sql import Window + >>> df = spark.createDataFrame( + ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) + >>> df.show() + +---+---+ + | c1| c2| + +---+---+ + | a| 1| + | a| 2| + | a| 3| + | b| 8| + | b| 2| + +---+---+ - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayofmonth('dt')).show() - +----------+----------+--------------+ - | dt|typeof(dt)|dayofmonth(dt)| - +----------+----------+--------------+ - |2015-04-08| date| 8| - |2024-10-31| date| 31| - +----------+----------+--------------+ + >>> w = Window.partitionBy("c1").orderBy("c2") + >>> df.withColumn("ntile", sf.ntile(2).over(w)).show() + +---+---+-----+ + | c1| c2|ntile| + +---+---+-----+ + | a| 1| 1| + | a| 2| 1| + | a| 3| 2| + | b| 2| 1| + | b| 8| 2| + +---+---+-----+ + """ + return _invoke_function("ntile", int(_enum_to_value(n))) - Example 4: Extract the day of the month from a timestamp column - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayofmonth('ts')).show() - +-------------------+----------+--------------+ - | ts|typeof(ts)|dayofmonth(ts)| - +-------------------+----------+--------------+ - |2015-04-08 13:08:15| timestamp| 8| - |2024-10-31 10:09:16| timestamp| 31| - +-------------------+----------+--------------+ - """ - return _invoke_function_over_columns("dayofmonth", col) +# ---------------------- Date/Timestamp functions ------------------------------ @_try_remote_functions -def day(col: "ColumnOrName") -> Column: +def curdate() -> Column: """ - Extract the day of the month of a given date/timestamp as integer. + Returns the current date at the start of query evaluation as a :class:`DateType` column. + All calls of current_date within the same query return the same value. .. versionadded:: 3.5.0 - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target date/timestamp column to work on. - A column that evaluates to a date, timestamp or string. - Returns ------- :class:`~pyspark.sql.Column` - day of the month for given date/timestamp as integer. - Returns a column that evaluates to an integer. + current date. See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.dayname` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.date_part` - :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.current_timestamp` + :meth:`pyspark.sql.functions.localtimestamp` Examples -------- - Example 1: Extract the day of the month from a string column representing dates + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.curdate()).show() # doctest: +SKIP + +--------------+ + |current_date()| + +--------------+ + | 2022-08-26| + +--------------+ + """ + return _invoke_function("curdate") - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.day('dt')).show() - +----------+----------+-------+ - | dt|typeof(dt)|day(dt)| - +----------+----------+-------+ - |2015-04-08| string| 8| - |2024-10-31| string| 31| - +----------+----------+-------+ - Example 2: Extract the day of the month from a string column representing timestamp +@_try_remote_functions +def current_date() -> Column: + """ + Returns the current date at the start of query evaluation as a :class:`DateType` column. + All calls of current_date within the same query return the same value. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.day('ts')).show() - +-------------------+----------+-------+ - | ts|typeof(ts)|day(ts)| - +-------------------+----------+-------+ - |2015-04-08 13:08:15| string| 8| - |2024-10-31 10:09:16| string| 31| - +-------------------+----------+-------+ + .. versionadded:: 1.5.0 - Example 3: Extract the day of the month from a date column + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.day('dt')).show() - +----------+----------+-------+ - | dt|typeof(dt)|day(dt)| - +----------+----------+-------+ - |2015-04-08| date| 8| - |2024-10-31| date| 31| - +----------+----------+-------+ + Returns + ------- + :class:`~pyspark.sql.Column` + current date. - Example 4: Extract the day of the month from a timestamp column + See Also + -------- + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.curdate` + :meth:`pyspark.sql.functions.current_timestamp` + :meth:`pyspark.sql.functions.localtimestamp` - >>> import datetime + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.day('ts')).show() - +-------------------+----------+-------+ - | ts|typeof(ts)|day(ts)| - +-------------------+----------+-------+ - |2015-04-08 13:08:15| timestamp| 8| - |2024-10-31 10:09:16| timestamp| 31| - +-------------------+----------+-------+ + >>> spark.range(1).select(sf.current_date()).show() # doctest: +SKIP + +--------------+ + |current_date()| + +--------------+ + | 2022-08-26| + +--------------+ """ - return _invoke_function_over_columns("day", col) + return _invoke_function("current_date") @_try_remote_functions -def dayofyear(col: "ColumnOrName") -> Column: +def current_timezone() -> Column: """ - Extract the day of the year of a given date/timestamp as integer. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Returns the current session local timezone. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target date/timestamp column to work on. - A column that evaluates to a date, timestamp or string. + .. versionadded:: 3.5.0 Returns ------- :class:`~pyspark.sql.Column` - day of the year for given date/timestamp as integer. - Returns a column that evaluates to an integer. + current session local timezone. See Also -------- - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.convert_timezone` Examples -------- - Example 1: Extract the day of the year from a string column representing dates + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayofyear('dt')).show() - +----------+----------+-------------+ - | dt|typeof(dt)|dayofyear(dt)| - +----------+----------+-------------+ - |2015-04-08| string| 98| - |2024-10-31| string| 305| - +----------+----------+-------------+ + >>> spark.range(1).select(sf.current_timezone()).show() + +-------------------+ + | current_timezone()| + +-------------------+ + |America/Los_Angeles| + +-------------------+ - Example 2: Extract the day of the year from a string column representing timestamp + Switch the timezone to Shanghai. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayofyear('ts')).show() - +-------------------+----------+-------------+ - | ts|typeof(ts)|dayofyear(ts)| - +-------------------+----------+-------------+ - |2015-04-08 13:08:15| string| 98| - |2024-10-31 10:09:16| string| 305| - +-------------------+----------+-------------+ + >>> spark.conf.set("spark.sql.session.timeZone", "Asia/Shanghai") + >>> spark.range(1).select(sf.current_timezone()).show() + +------------------+ + |current_timezone()| + +------------------+ + | Asia/Shanghai| + +------------------+ - Example 3: Extract the day of the year from a date column + >>> spark.conf.unset("spark.sql.session.timeZone") + """ + return _invoke_function("current_timezone") - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.date(2015, 4, 8),), - ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayofyear('dt')).show() - +----------+----------+-------------+ - | dt|typeof(dt)|dayofyear(dt)| - +----------+----------+-------------+ - |2015-04-08| date| 98| - |2024-10-31| date| 305| - +----------+----------+-------------+ - Example 4: Extract the day of the year from a timestamp column +@overload +def current_time() -> Column: ... - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayofyear('ts')).show() - +-------------------+----------+-------------+ - | ts|typeof(ts)|dayofyear(ts)| - +-------------------+----------+-------------+ - |2015-04-08 13:08:15| timestamp| 98| - |2024-10-31 10:09:16| timestamp| 305| - +-------------------+----------+-------------+ - """ - return _invoke_function_over_columns("dayofyear", col) + +@overload +def current_time(precision: int) -> Column: ... @_try_remote_functions -def hour(col: "ColumnOrName") -> Column: +def current_time(precision: Optional[int] = None) -> Column: """ - Extract the hours of a given timestamp as integer. - - .. versionadded:: 1.5.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Returns the current time at the start of query evaluation as a :class:`TimeType` column. All + calls of current_time within the same query return the same value. - .. versionchanged:: 4.1.0 - Added support for time type. + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target date/time/timestamp column to work on. - A column that evaluates to a timestamp or time. + precision: literal int, optional + number in the range [0..6], indicating how many fractional digits of seconds to include. + If omitted, the default is 6. Returns ------- :class:`~pyspark.sql.Column` - hour part of the timestamp as integer. - Returns a column that evaluates to an integer. + current time. See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.current_timestamp` Examples -------- - Example 1: Extract the hours from a string column representing timestamp - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.hour('ts')).show() - +-------------------+----------+--------+ - | ts|typeof(ts)|hour(ts)| - +-------------------+----------+--------+ - |2015-04-08 13:08:15| string| 13| - |2024-10-31 10:09:16| string| 10| - +-------------------+----------+--------+ - - Example 2: Extract the hours from a timestamp column + Example 1: Current time with default precision - >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.hour('ts')).show() - +-------------------+----------+--------+ - | ts|typeof(ts)|hour(ts)| - +-------------------+----------+--------+ - |2015-04-08 13:08:15| timestamp| 13| - |2024-10-31 10:09:16| timestamp| 10| - +-------------------+----------+--------+ + >>> spark.range(1).select(sf.current_time().alias("time")).show() # doctest: +SKIP + +---------------+ + | time| + +---------------+ + |16:57:04.304361| + +---------------+ - Example 3: Extract the hours from a time column + Example 2: Current time with specified precision - >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... ("13:08:15",), - ... ("10:09:16",)], ['t']).withColumn("t", sf.col("t").cast("time")) - >>> df.select("*", sf.typeof('t'), sf.hour('t')).show() - +--------+---------+-------+ - | t|typeof(t)|hour(t)| - +--------+---------+-------+ - |13:08:15| time(6)| 13| - |10:09:16| time(6)| 10| - +--------+---------+-------+ + >>> spark.range(1).select(sf.current_time(3).alias("time")).show() # doctest: +SKIP + +------------+ + | time| + +------------+ + |16:57:04.304| + +------------+ """ - return _invoke_function_over_columns("hour", col) + if precision is None: + return _invoke_function("current_time") + else: + return _invoke_function("current_time", _enum_to_value(precision)) @_try_remote_functions -def minute(col: "ColumnOrName") -> Column: +def current_timestamp() -> Column: """ - Extract the minutes of a given timestamp as integer. + Returns the current timestamp at the start of query evaluation as a :class:`TimestampType` + column. All calls of current_timestamp within the same query return the same value. .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.1.0 - Added support for time type. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target date/time/timestamp column to work on. - A column that evaluates to a timestamp or time. - - See Also - -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.date_part` - Returns ------- :class:`~pyspark.sql.Column` - minutes part of the timestamp as integer. - Returns a column that evaluates to an integer. + current date and time. - Examples + See Also -------- - Example 1: Extract the minutes from a string column representing timestamp + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.curdate` + :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.localtimestamp` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.minute('ts')).show() - +-------------------+----------+----------+ - | ts|typeof(ts)|minute(ts)| - +-------------------+----------+----------+ - |2015-04-08 13:08:15| string| 8| - |2024-10-31 10:09:16| string| 9| - +-------------------+----------+----------+ + >>> spark.range(1).select(sf.current_timestamp()).show(truncate=False) # doctest: +SKIP + +-----------------------+ + |current_timestamp() | + +-----------------------+ + |2022-08-26 21:23:22.716| + +-----------------------+ + """ + return _invoke_function("current_timestamp") - Example 2: Extract the minutes from a timestamp column - >>> import datetime - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.minute('ts')).show() - +-------------------+----------+----------+ - | ts|typeof(ts)|minute(ts)| - +-------------------+----------+----------+ - |2015-04-08 13:08:15| timestamp| 8| - |2024-10-31 10:09:16| timestamp| 9| - +-------------------+----------+----------+ +@_try_remote_functions +def now() -> Column: + """ + Returns the current timestamp at the start of query evaluation. - Example 3: Extract the minutes from a time column + .. versionadded:: 3.5.0 - >>> import datetime + Returns + ------- + :class:`~pyspark.sql.Column` + current timestamp at the start of query evaluation. + + See Also + -------- + :meth:`pyspark.sql.functions.curdate` + :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.current_timestamp` + :meth:`pyspark.sql.functions.localtimestamp` + + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... ("13:08:15",), - ... ("10:09:16",)], ['t']).withColumn("t", sf.col("t").cast("time")) - >>> df.select("*", sf.typeof('t'), sf.minute('t')).show() - +--------+---------+---------+ - | t|typeof(t)|minute(t)| - +--------+---------+---------+ - |13:08:15| time(6)| 8| - |10:09:16| time(6)| 9| - +--------+---------+---------+ + >>> spark.range(1).select(sf.now()).show(truncate=False) # doctest: +SKIP + +--------------------------+ + |now() | + +--------------------------+ + |2023-12-08 15:18:18.482269| + +--------------------------+ """ - return _invoke_function_over_columns("minute", col) + return _invoke_function("now") @_try_remote_functions -def second(col: "ColumnOrName") -> Column: +def localtimestamp() -> Column: """ - Extract the seconds of a given date as integer. + Returns the current timestamp without time zone at the start of query evaluation + as a timestamp without time zone column. All calls of localtimestamp within the + same query return the same value. + + .. versionadded:: 3.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Returns + ------- + :class:`~pyspark.sql.Column` + current local date and time. + + See Also + -------- + :meth:`pyspark.sql.functions.now` + :meth:`pyspark.sql.functions.curdate` + :meth:`pyspark.sql.functions.current_date` + :meth:`pyspark.sql.functions.current_timestamp` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> spark.range(1).select(sf.localtimestamp()).show(truncate=False) # doctest: +SKIP + +-----------------------+ + |localtimestamp() | + +-----------------------+ + |2022-08-26 21:28:34.639| + +-----------------------+ + """ + return _invoke_function("localtimestamp") + + +@_try_remote_functions +def date_format(date: "ColumnOrName", format: str) -> Column: + """ + Converts a date/timestamp/string to a value of string in the format specified by the date + format given by the second argument. + + A pattern could be for instance `dd.MM.yyyy` and could return a string like '18.03.1993'. All + pattern letters of `datetime pattern`_. can be used. + + .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.1.0 - Added support for time type. + Notes + ----- + Whenever possible, use specialized functions like `year`. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target date/time/timestamp column to work on. + date : :class:`~pyspark.sql.Column` or column name + input column of values to format. A column that evaluates to a timestamp or time. + format: literal string + format to use to represent datetime values. + A column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.to_date` + :meth:`pyspark.sql.functions.to_timestamp` + :meth:`pyspark.sql.functions.to_timestamp_ltz` + :meth:`pyspark.sql.functions.to_timestamp_ntz` + :meth:`pyspark.sql.functions.to_utc_timestamp` + :meth:`pyspark.sql.functions.try_to_timestamp` Returns ------- :class:`~pyspark.sql.Column` - `seconds` part of the timestamp as integer. - Returns a column that evaluates to an integer. - - See Also - -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.extract` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.date_part` + string value representing formatted datetime. + Returns a column that evaluates to a string. Examples -------- - Example 1: Extract the seconds from a string column representing timestamp + Example 1: Format a string column representing dates + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.date_format('dt', 'MM/dd/yyyy')).show() + +----------+----------+---------------------------+ + | dt|typeof(dt)|date_format(dt, MM/dd/yyyy)| + +----------+----------+---------------------------+ + |2015-04-08| string| 04/08/2015| + |2024-10-31| string| 10/31/2024| + +----------+----------+---------------------------+ + + Example 2: Format a string column representing timestamp >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.second('ts')).show() - +-------------------+----------+----------+ - | ts|typeof(ts)|second(ts)| - +-------------------+----------+----------+ - |2015-04-08 13:08:15| string| 15| - |2024-10-31 10:09:16| string| 16| - +-------------------+----------+----------+ + >>> df.select("*", sf.typeof('ts'), sf.date_format('ts', 'yy=MM=dd HH=mm=ss')).show() + +-------------------+----------+----------------------------------+ + | ts|typeof(ts)|date_format(ts, yy=MM=dd HH=mm=ss)| + +-------------------+----------+----------------------------------+ + |2015-04-08 13:08:15| string| 15=04=08 13=08=15| + |2024-10-31 10:09:16| string| 24=10=31 10=09=16| + +-------------------+----------+----------------------------------+ - Example 2: Extract the seconds from a timestamp column + Example 3: Format a date column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ - ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), - ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.second('ts')).show() - +-------------------+----------+----------+ - | ts|typeof(ts)|second(ts)| - +-------------------+----------+----------+ - |2015-04-08 13:08:15| timestamp| 15| - |2024-10-31 10:09:16| timestamp| 16| - +-------------------+----------+----------+ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.date_format('dt', 'yy--MM--dd')).show() + +----------+----------+---------------------------+ + | dt|typeof(dt)|date_format(dt, yy--MM--dd)| + +----------+----------+---------------------------+ + |2015-04-08| date| 15--04--08| + |2024-10-31| date| 24--10--31| + +----------+----------+---------------------------+ - Example 3: Extract the seconds from a time column + Example 4: Format a timestamp column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ - ... ("13:08:15",), - ... ("10:09:16",)], ['t']).withColumn("t", sf.col("t").cast("time")) - >>> df.select("*", sf.typeof('t'), sf.second('t')).show() - +--------+---------+---------+ - | t|typeof(t)|second(t)| - +--------+---------+---------+ - |13:08:15| time(6)| 15| - |10:09:16| time(6)| 16| - +--------+---------+---------+ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.date_format('ts', 'yy=MM=dd HH=mm=ss')).show() + +-------------------+----------+----------------------------------+ + | ts|typeof(ts)|date_format(ts, yy=MM=dd HH=mm=ss)| + +-------------------+----------+----------------------------------+ + |2015-04-08 13:08:15| timestamp| 15=04=08 13=08=15| + |2024-10-31 10:09:16| timestamp| 24=10=31 10=09=16| + +-------------------+----------+----------------------------------+ """ - return _invoke_function_over_columns("second", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("date_format", _to_java_column(date), _enum_to_value(format)) @_try_remote_functions -def weekofyear(col: "ColumnOrName") -> Column: +def year(col: "ColumnOrName") -> Column: """ - Extract the week number of a given date as integer. - A week is considered to start on a Monday and week 1 is the first week with more than 3 days, - as defined by ISO 8601 + Extract the year of a given date/timestamp as integer. .. versionadded:: 1.5.0 @@ -10523,87 +10217,95 @@ def weekofyear(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target timestamp column to work on. + target date/timestamp column to work on. A column that evaluates to a date, timestamp or string. Returns ------- :class:`~pyspark.sql.Column` - `week` of the year for given date as integer. + year part of the date/timestamp as integer. Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.weekday` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.dayofmonth` - :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.date_part` Examples -------- - Example 1: Extract the week of the year from a string column representing dates + Example 1: Extract the year from a string column representing dates >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.weekofyear('dt')).show() - +----------+----------+--------------+ - | dt|typeof(dt)|weekofyear(dt)| - +----------+----------+--------------+ - |2015-04-08| string| 15| - |2024-10-31| string| 44| - +----------+----------+--------------+ + >>> df.select("*", sf.typeof('dt'), sf.year('dt')).show() + +----------+----------+--------+ + | dt|typeof(dt)|year(dt)| + +----------+----------+--------+ + |2015-04-08| string| 2015| + |2024-10-31| string| 2024| + +----------+----------+--------+ - Example 2: Extract the week of the year from a string column representing timestamp + Example 2: Extract the year from a string column representing timestamp >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.weekofyear('ts')).show() - +-------------------+----------+--------------+ - | ts|typeof(ts)|weekofyear(ts)| - +-------------------+----------+--------------+ - |2015-04-08 13:08:15| string| 15| - |2024-10-31 10:09:16| string| 44| - +-------------------+----------+--------------+ + >>> df.select("*", sf.typeof('ts'), sf.year('ts')).show() + +-------------------+----------+--------+ + | ts|typeof(ts)|year(ts)| + +-------------------+----------+--------+ + |2015-04-08 13:08:15| string| 2015| + |2024-10-31 10:09:16| string| 2024| + +-------------------+----------+--------+ - Example 3: Extract the week of the year from a date column + Example 3: Extract the year from a date column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.date(2015, 4, 8),), ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.weekofyear('dt')).show() - +----------+----------+--------------+ - | dt|typeof(dt)|weekofyear(dt)| - +----------+----------+--------------+ - |2015-04-08| date| 15| - |2024-10-31| date| 44| - +----------+----------+--------------+ + >>> df.select("*", sf.typeof('dt'), sf.year('dt')).show() + +----------+----------+--------+ + | dt|typeof(dt)|year(dt)| + +----------+----------+--------+ + |2015-04-08| date| 2015| + |2024-10-31| date| 2024| + +----------+----------+--------+ - Example 4: Extract the week of the year from a timestamp column + Example 4: Extract the year from a timestamp column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.weekofyear('ts')).show() - +-------------------+----------+--------------+ - | ts|typeof(ts)|weekofyear(ts)| - +-------------------+----------+--------------+ - |2015-04-08 13:08:15| timestamp| 15| - |2024-10-31 10:09:16| timestamp| 44| - +-------------------+----------+--------------+ + >>> df.select("*", sf.typeof('ts'), sf.year('ts')).show() + +-------------------+----------+--------+ + | ts|typeof(ts)|year(ts)| + +-------------------+----------+--------+ + |2015-04-08 13:08:15| timestamp| 2015| + |2024-10-31 10:09:16| timestamp| 2024| + +-------------------+----------+--------+ """ - return _invoke_function_over_columns("weekofyear", col) + return _invoke_function_over_columns("year", col) @_try_remote_functions -def weekday(col: "ColumnOrName") -> Column: +def quarter(col: "ColumnOrName") -> Column: """ - Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + Extract the quarter of a given date/timestamp as integer. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- @@ -10614,82 +10316,89 @@ def weekday(col: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + quarter of the date/timestamp as integer. Returns a column that evaluates to an integer. See Also -------- + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.weekofyear` - :meth:`pyspark.sql.functions.dayofweek` - :meth:`pyspark.sql.functions.dayofyear` - :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.date_part` Examples -------- - Example 1: Extract the day of the week from a string column representing dates + Example 1: Extract the quarter from a string column representing dates >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.weekday('dt')).show() + >>> df.select("*", sf.typeof('dt'), sf.quarter('dt')).show() +----------+----------+-----------+ - | dt|typeof(dt)|weekday(dt)| + | dt|typeof(dt)|quarter(dt)| +----------+----------+-----------+ |2015-04-08| string| 2| - |2024-10-31| string| 3| + |2024-10-31| string| 4| +----------+----------+-----------+ - Example 2: Extract the day of the week from a string column representing timestamp + Example 2: Extract the quarter from a string column representing timestamp >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.weekday('ts')).show() + >>> df.select("*", sf.typeof('ts'), sf.quarter('ts')).show() +-------------------+----------+-----------+ - | ts|typeof(ts)|weekday(ts)| + | ts|typeof(ts)|quarter(ts)| +-------------------+----------+-----------+ |2015-04-08 13:08:15| string| 2| - |2024-10-31 10:09:16| string| 3| + |2024-10-31 10:09:16| string| 4| +-------------------+----------+-----------+ - Example 3: Extract the day of the week from a date column + Example 3: Extract the quarter from a date column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.date(2015, 4, 8),), ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.weekday('dt')).show() + >>> df.select("*", sf.typeof('dt'), sf.quarter('dt')).show() +----------+----------+-----------+ - | dt|typeof(dt)|weekday(dt)| + | dt|typeof(dt)|quarter(dt)| +----------+----------+-----------+ |2015-04-08| date| 2| - |2024-10-31| date| 3| + |2024-10-31| date| 4| +----------+----------+-----------+ - Example 4: Extract the day of the week from a timestamp column + Example 4: Extract the quarter from a timestamp column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.weekday('ts')).show() + >>> df.select("*", sf.typeof('ts'), sf.quarter('ts')).show() +-------------------+----------+-----------+ - | ts|typeof(ts)|weekday(ts)| + | ts|typeof(ts)|quarter(ts)| +-------------------+----------+-----------+ |2015-04-08 13:08:15| timestamp| 2| - |2024-10-31 10:09:16| timestamp| 3| + |2024-10-31 10:09:16| timestamp| 4| +-------------------+----------+-----------+ """ - return _invoke_function_over_columns("weekday", col) + return _invoke_function_over_columns("quarter", col) @_try_remote_functions -def monthname(col: "ColumnOrName") -> Column: +def month(col: "ColumnOrName") -> Column: """ - Returns the three-letter abbreviated month name from the given date. + Extract the month of a given date/timestamp as integer. - .. versionadded:: 4.0.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- @@ -10700,79 +10409,91 @@ def monthname(col: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - the three-letter abbreviation of month name for date/timestamp (Jan, Feb, Mar...) - Returns a column that evaluates to a string. + month part of the date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.dayname` + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.monthname` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.date_part` Examples -------- - Example 1: Extract the month name from a string column representing dates + Example 1: Extract the month from a string column representing dates >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.monthname('dt')).show() - +----------+----------+-------------+ - | dt|typeof(dt)|monthname(dt)| - +----------+----------+-------------+ - |2015-04-08| string| Apr| - |2024-10-31| string| Oct| - +----------+----------+-------------+ + >>> df.select("*", sf.typeof('dt'), sf.month('dt')).show() + +----------+----------+---------+ + | dt|typeof(dt)|month(dt)| + +----------+----------+---------+ + |2015-04-08| string| 4| + |2024-10-31| string| 10| + +----------+----------+---------+ - Example 2: Extract the month name from a string column representing timestamp + Example 2: Extract the month from a string column representing timestamp >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.monthname('ts')).show() - +-------------------+----------+-------------+ - | ts|typeof(ts)|monthname(ts)| - +-------------------+----------+-------------+ - |2015-04-08 13:08:15| string| Apr| - |2024-10-31 10:09:16| string| Oct| - +-------------------+----------+-------------+ + >>> df.select("*", sf.typeof('ts'), sf.month('ts')).show() + +-------------------+----------+---------+ + | ts|typeof(ts)|month(ts)| + +-------------------+----------+---------+ + |2015-04-08 13:08:15| string| 4| + |2024-10-31 10:09:16| string| 10| + +-------------------+----------+---------+ - Example 3: Extract the month name from a date column + Example 3: Extract the month from a date column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.date(2015, 4, 8),), ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.monthname('dt')).show() - +----------+----------+-------------+ - | dt|typeof(dt)|monthname(dt)| - +----------+----------+-------------+ - |2015-04-08| date| Apr| - |2024-10-31| date| Oct| - +----------+----------+-------------+ + >>> df.select("*", sf.typeof('dt'), sf.month('dt')).show() + +----------+----------+---------+ + | dt|typeof(dt)|month(dt)| + +----------+----------+---------+ + |2015-04-08| date| 4| + |2024-10-31| date| 10| + +----------+----------+---------+ - Example 4: Extract the month name from a timestamp column + Example 3: Extract the month from a timestamp column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.monthname('ts')).show() - +-------------------+----------+-------------+ - | ts|typeof(ts)|monthname(ts)| - +-------------------+----------+-------------+ - |2015-04-08 13:08:15| timestamp| Apr| - |2024-10-31 10:09:16| timestamp| Oct| - +-------------------+----------+-------------+ + >>> df.select("*", sf.typeof('ts'), sf.month('ts')).show() + +-------------------+----------+---------+ + | ts|typeof(ts)|month(ts)| + +-------------------+----------+---------+ + |2015-04-08 13:08:15| timestamp| 4| + |2024-10-31 10:09:16| timestamp| 10| + +-------------------+----------+---------+ """ - return _invoke_function_over_columns("monthname", col) + return _invoke_function_over_columns("month", col) @_try_remote_functions -def dayname(col: "ColumnOrName") -> Column: +def dayofweek(col: "ColumnOrName") -> Column: """ - Date and Timestamp Function: Returns the three-letter abbreviated day name from the given date. + Extract the day of the week of a given date/timestamp as integer. + Ranges from 1 for a Sunday through to 7 for a Saturday - .. versionadded:: 4.0.0 + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- @@ -10783,207 +10504,371 @@ def dayname(col: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - the three-letter abbreviation of day name for date/timestamp (Mon, Tue, Wed...) - Returns a column that evaluates to a string. + day of the week for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.monthname` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.weekofyear` Examples -------- - Example 1: Extract the weekday name from a string column representing dates + Example 1: Extract the day of the week from a string column representing dates >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayname('dt')).show() - +----------+----------+-----------+ - | dt|typeof(dt)|dayname(dt)| - +----------+----------+-----------+ - |2015-04-08| string| Wed| - |2024-10-31| string| Thu| - +----------+----------+-----------+ + >>> df.select("*", sf.typeof('dt'), sf.dayofweek('dt')).show() + +----------+----------+-------------+ + | dt|typeof(dt)|dayofweek(dt)| + +----------+----------+-------------+ + |2015-04-08| string| 4| + |2024-10-31| string| 5| + +----------+----------+-------------+ - Example 2: Extract the weekday name from a string column representing timestamp + Example 2: Extract the day of the week from a string column representing timestamp >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayname('ts')).show() - +-------------------+----------+-----------+ - | ts|typeof(ts)|dayname(ts)| - +-------------------+----------+-----------+ - |2015-04-08 13:08:15| string| Wed| - |2024-10-31 10:09:16| string| Thu| - +-------------------+----------+-----------+ + >>> df.select("*", sf.typeof('ts'), sf.dayofweek('ts')).show() + +-------------------+----------+-------------+ + | ts|typeof(ts)|dayofweek(ts)| + +-------------------+----------+-------------+ + |2015-04-08 13:08:15| string| 4| + |2024-10-31 10:09:16| string| 5| + +-------------------+----------+-------------+ - Example 3: Extract the weekday name from a date column + Example 3: Extract the day of the week from a date column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.date(2015, 4, 8),), ... (datetime.date(2024, 10, 31),)], ['dt']) - >>> df.select("*", sf.typeof('dt'), sf.dayname('dt')).show() - +----------+----------+-----------+ - | dt|typeof(dt)|dayname(dt)| - +----------+----------+-----------+ - |2015-04-08| date| Wed| - |2024-10-31| date| Thu| - +----------+----------+-----------+ + >>> df.select("*", sf.typeof('dt'), sf.dayofweek('dt')).show() + +----------+----------+-------------+ + | dt|typeof(dt)|dayofweek(dt)| + +----------+----------+-------------+ + |2015-04-08| date| 4| + |2024-10-31| date| 5| + +----------+----------+-------------+ - Example 4: Extract the weekday name from a timestamp column + Example 4: Extract the day of the week from a timestamp column >>> import datetime >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([ ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) - >>> df.select("*", sf.typeof('ts'), sf.dayname('ts')).show() - +-------------------+----------+-----------+ - | ts|typeof(ts)|dayname(ts)| - +-------------------+----------+-----------+ - |2015-04-08 13:08:15| timestamp| Wed| - |2024-10-31 10:09:16| timestamp| Thu| - +-------------------+----------+-----------+ + >>> df.select("*", sf.typeof('ts'), sf.dayofweek('ts')).show() + +-------------------+----------+-------------+ + | ts|typeof(ts)|dayofweek(ts)| + +-------------------+----------+-------------+ + |2015-04-08 13:08:15| timestamp| 4| + |2024-10-31 10:09:16| timestamp| 5| + +-------------------+----------+-------------+ """ - return _invoke_function_over_columns("dayname", col) + return _invoke_function_over_columns("dayofweek", col) @_try_remote_functions -def extract(field: Column, source: "ColumnOrName") -> Column: +def dayofmonth(col: "ColumnOrName") -> Column: """ - Extracts a part of the date/timestamp or interval source. + Extract the day of the month of a given date/timestamp as integer. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target date/timestamp column to work on. + A column that evaluates to a date, timestamp or string. + + See Also + -------- + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.weekofyear` + + Returns + ------- + :class:`~pyspark.sql.Column` + day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. + + Examples + -------- + Example 1: Extract the day of the month from a string column representing dates + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.dayofmonth('dt')).show() + +----------+----------+--------------+ + | dt|typeof(dt)|dayofmonth(dt)| + +----------+----------+--------------+ + |2015-04-08| string| 8| + |2024-10-31| string| 31| + +----------+----------+--------------+ + + Example 2: Extract the day of the month from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.dayofmonth('ts')).show() + +-------------------+----------+--------------+ + | ts|typeof(ts)|dayofmonth(ts)| + +-------------------+----------+--------------+ + |2015-04-08 13:08:15| string| 8| + |2024-10-31 10:09:16| string| 31| + +-------------------+----------+--------------+ + + Example 3: Extract the day of the month from a date column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.dayofmonth('dt')).show() + +----------+----------+--------------+ + | dt|typeof(dt)|dayofmonth(dt)| + +----------+----------+--------------+ + |2015-04-08| date| 8| + |2024-10-31| date| 31| + +----------+----------+--------------+ + + Example 4: Extract the day of the month from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.dayofmonth('ts')).show() + +-------------------+----------+--------------+ + | ts|typeof(ts)|dayofmonth(ts)| + +-------------------+----------+--------------+ + |2015-04-08 13:08:15| timestamp| 8| + |2024-10-31 10:09:16| timestamp| 31| + +-------------------+----------+--------------+ + """ + return _invoke_function_over_columns("dayofmonth", col) + + +@_try_remote_functions +def day(col: "ColumnOrName") -> Column: + """ + Extract the day of the month of a given date/timestamp as integer. .. versionadded:: 3.5.0 Parameters ---------- - field : :class:`~pyspark.sql.Column` - selects which part of the source should be extracted. - source : :class:`~pyspark.sql.Column` or column name - a date, time, timestamp, or interval column from where `field` should be extracted. + col : :class:`~pyspark.sql.Column` or column name + target date/timestamp column to work on. + A column that evaluates to a date, timestamp or string. Returns ------- :class:`~pyspark.sql.Column` - a part of the date/timestamp or interval source. - Returns a column whose type depends on the field to extract, e.g. an integer - for ``YEAR`` and a decimal for ``SECOND``. + day of the month for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- :meth:`pyspark.sql.functions.year` :meth:`pyspark.sql.functions.quarter` :meth:`pyspark.sql.functions.month` - :meth:`pyspark.sql.functions.day` :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.dayname` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.extract` :meth:`pyspark.sql.functions.datepart` :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.weekday` Examples -------- + Example 1: Extract the day of the month from a string column representing dates + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.day('dt')).show() + +----------+----------+-------+ + | dt|typeof(dt)|day(dt)| + +----------+----------+-------+ + |2015-04-08| string| 8| + |2024-10-31| string| 31| + +----------+----------+-------+ + + Example 2: Extract the day of the month from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.day('ts')).show() + +-------------------+----------+-------+ + | ts|typeof(ts)|day(ts)| + +-------------------+----------+-------+ + |2015-04-08 13:08:15| string| 8| + |2024-10-31 10:09:16| string| 31| + +-------------------+----------+-------+ + + Example 3: Extract the day of the month from a date column + >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts']) - >>> df.select( - ... '*', - ... sf.extract(sf.lit('YEAR'), 'ts').alias('year'), - ... sf.extract(sf.lit('month'), 'ts').alias('month'), - ... sf.extract(sf.lit('WEEK'), 'ts').alias('week'), - ... sf.extract(sf.lit('D'), df.ts).alias('day'), - ... sf.extract(sf.lit('M'), df.ts).alias('minute'), - ... sf.extract(sf.lit('S'), df.ts).alias('second') - ... ).show() - +-------------------+----+-----+----+---+------+---------+ - | ts|year|month|week|day|minute| second| - +-------------------+----+-----+----+---+------+---------+ - |2015-04-08 13:08:15|2015| 4| 15| 8| 8|15.000000| - +-------------------+----+-----+----+---+------+---------+ + >>> df = spark.createDataFrame([ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.day('dt')).show() + +----------+----------+-------+ + | dt|typeof(dt)|day(dt)| + +----------+----------+-------+ + |2015-04-08| date| 8| + |2024-10-31| date| 31| + +----------+----------+-------+ + + Example 4: Extract the day of the month from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.day('ts')).show() + +-------------------+----------+-------+ + | ts|typeof(ts)|day(ts)| + +-------------------+----------+-------+ + |2015-04-08 13:08:15| timestamp| 8| + |2024-10-31 10:09:16| timestamp| 31| + +-------------------+----------+-------+ """ - return _invoke_function_over_columns("extract", field, source) + return _invoke_function_over_columns("day", col) @_try_remote_functions -def date_part(field: Column, source: "ColumnOrName") -> Column: +def dayofyear(col: "ColumnOrName") -> Column: """ - Extracts a part of the date/timestamp or interval source. + Extract the day of the year of a given date/timestamp as integer. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - field : :class:`~pyspark.sql.Column` - selects which part of the source should be extracted, and supported string values - are as same as the fields of the equivalent function `extract`. - source : :class:`~pyspark.sql.Column` or column name - a date, time, timestamp, or interval column from where `field` should be extracted. + col : :class:`~pyspark.sql.Column` or column name + target date/timestamp column to work on. + A column that evaluates to a date, timestamp or string. Returns ------- :class:`~pyspark.sql.Column` - a part of the date/timestamp or interval source. - Returns a column whose type depends on the field to extract, e.g. an integer - for ``YEAR`` and a decimal for ``SECOND``. + day of the year for given date/timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.year` - :meth:`pyspark.sql.functions.quarter` - :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` - :meth:`pyspark.sql.functions.minute` - :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.datepart` - :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` Examples -------- + Example 1: Extract the day of the year from a string column representing dates + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.dayofyear('dt')).show() + +----------+----------+-------------+ + | dt|typeof(dt)|dayofyear(dt)| + +----------+----------+-------------+ + |2015-04-08| string| 98| + |2024-10-31| string| 305| + +----------+----------+-------------+ + + Example 2: Extract the day of the year from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.dayofyear('ts')).show() + +-------------------+----------+-------------+ + | ts|typeof(ts)|dayofyear(ts)| + +-------------------+----------+-------------+ + |2015-04-08 13:08:15| string| 98| + |2024-10-31 10:09:16| string| 305| + +-------------------+----------+-------------+ + + Example 3: Extract the day of the year from a date column + >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts']) - >>> df.select( - ... '*', - ... sf.date_part(sf.lit('YEAR'), 'ts').alias('year'), - ... sf.date_part(sf.lit('month'), 'ts').alias('month'), - ... sf.date_part(sf.lit('WEEK'), 'ts').alias('week'), - ... sf.date_part(sf.lit('D'), df.ts).alias('day'), - ... sf.date_part(sf.lit('M'), df.ts).alias('minute'), - ... sf.date_part(sf.lit('S'), df.ts).alias('second') - ... ).show() - +-------------------+----+-----+----+---+------+---------+ - | ts|year|month|week|day|minute| second| - +-------------------+----+-----+----+---+------+---------+ - |2015-04-08 13:08:15|2015| 4| 15| 8| 8|15.000000| - +-------------------+----+-----+----+---+------+---------+ + >>> df = spark.createDataFrame([ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.dayofyear('dt')).show() + +----------+----------+-------------+ + | dt|typeof(dt)|dayofyear(dt)| + +----------+----------+-------------+ + |2015-04-08| date| 98| + |2024-10-31| date| 305| + +----------+----------+-------------+ + + Example 4: Extract the day of the year from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.dayofyear('ts')).show() + +-------------------+----------+-------------+ + | ts|typeof(ts)|dayofyear(ts)| + +-------------------+----------+-------------+ + |2015-04-08 13:08:15| timestamp| 98| + |2024-10-31 10:09:16| timestamp| 305| + +-------------------+----------+-------------+ """ - return _invoke_function_over_columns("date_part", field, source) + return _invoke_function_over_columns("dayofyear", col) @_try_remote_functions -def datepart(field: Column, source: "ColumnOrName") -> Column: +def hour(col: "ColumnOrName") -> Column: """ - Extracts a part of the date/timestamp or interval source. + Extract the hours of a given timestamp as integer. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. versionchanged:: 4.1.0 + Added support for time type. Parameters ---------- - field : :class:`~pyspark.sql.Column` - selects which part of the source should be extracted, and supported string values - are as same as the fields of the equivalent function `extract`. - source : :class:`~pyspark.sql.Column` or column name - a date, time, timestamp, or interval column from where `field` should be extracted. + col : :class:`~pyspark.sql.Column` or column name + target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` - a part of the date/timestamp or interval source. - Returns a column whose type depends on the field to extract, e.g. an integer - for ``YEAR`` and a decimal for ``SECOND``. + hour part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- @@ -10991,799 +10876,1503 @@ def datepart(field: Column, source: "ColumnOrName") -> Column: :meth:`pyspark.sql.functions.quarter` :meth:`pyspark.sql.functions.month` :meth:`pyspark.sql.functions.day` - :meth:`pyspark.sql.functions.hour` :meth:`pyspark.sql.functions.minute` :meth:`pyspark.sql.functions.second` - :meth:`pyspark.sql.functions.date_part` :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.date_part` Examples -------- + Example 1: Extract the hours from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.hour('ts')).show() + +-------------------+----------+--------+ + | ts|typeof(ts)|hour(ts)| + +-------------------+----------+--------+ + |2015-04-08 13:08:15| string| 13| + |2024-10-31 10:09:16| string| 10| + +-------------------+----------+--------+ + + Example 2: Extract the hours from a timestamp column + >>> import datetime >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts']) - >>> df.select( - ... '*', - ... sf.datepart(sf.lit('YEAR'), 'ts').alias('year'), - ... sf.datepart(sf.lit('month'), 'ts').alias('month'), - ... sf.datepart(sf.lit('WEEK'), 'ts').alias('week'), - ... sf.datepart(sf.lit('D'), df.ts).alias('day'), - ... sf.datepart(sf.lit('M'), df.ts).alias('minute'), - ... sf.datepart(sf.lit('S'), df.ts).alias('second') - ... ).show() - +-------------------+----+-----+----+---+------+---------+ - | ts|year|month|week|day|minute| second| - +-------------------+----+-----+----+---+------+---------+ - |2015-04-08 13:08:15|2015| 4| 15| 8| 8|15.000000| - +-------------------+----+-----+----+---+------+---------+ + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.hour('ts')).show() + +-------------------+----------+--------+ + | ts|typeof(ts)|hour(ts)| + +-------------------+----------+--------+ + |2015-04-08 13:08:15| timestamp| 13| + |2024-10-31 10:09:16| timestamp| 10| + +-------------------+----------+--------+ + + Example 3: Extract the hours from a time column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... ("13:08:15",), + ... ("10:09:16",)], ['t']).withColumn("t", sf.col("t").cast("time")) + >>> df.select("*", sf.typeof('t'), sf.hour('t')).show() + +--------+---------+-------+ + | t|typeof(t)|hour(t)| + +--------+---------+-------+ + |13:08:15| time(6)| 13| + |10:09:16| time(6)| 10| + +--------+---------+-------+ """ - return _invoke_function_over_columns("datepart", field, source) + return _invoke_function_over_columns("hour", col) @_try_remote_functions -def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName") -> Column: +def minute(col: "ColumnOrName") -> Column: """ - Returns a column with a date built from the year, month and day columns. + Extract the minutes of a given timestamp as integer. - .. versionadded:: 3.3.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.1.0 + Added support for time type. + Parameters ---------- - year : :class:`~pyspark.sql.Column` or column name - The year to build the date. - A column that evaluates to an integer. - month : :class:`~pyspark.sql.Column` or column name - The month to build the date. - A column that evaluates to an integer. - day : :class:`~pyspark.sql.Column` or column name - The day to build the date. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. + + See Also + -------- + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.date_part` Returns ------- :class:`~pyspark.sql.Column` - a date built from given parts. - Returns a column that evaluates to a date. - - See Also - -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` + minutes part of the timestamp as integer. + Returns a column that evaluates to an integer. Examples -------- + Example 1: Extract the minutes from a string column representing timestamp + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(2020, 6, 26)], ['Y', 'M', 'D']) - >>> df.select('*', sf.make_date(df.Y, 'M', df.D)).show() - +----+---+---+------------------+ - | Y| M| D|make_date(Y, M, D)| - +----+---+---+------------------+ - |2020| 6| 26| 2020-06-26| - +----+---+---+------------------+ + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.minute('ts')).show() + +-------------------+----------+----------+ + | ts|typeof(ts)|minute(ts)| + +-------------------+----------+----------+ + |2015-04-08 13:08:15| string| 8| + |2024-10-31 10:09:16| string| 9| + +-------------------+----------+----------+ + + Example 2: Extract the minutes from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.minute('ts')).show() + +-------------------+----------+----------+ + | ts|typeof(ts)|minute(ts)| + +-------------------+----------+----------+ + |2015-04-08 13:08:15| timestamp| 8| + |2024-10-31 10:09:16| timestamp| 9| + +-------------------+----------+----------+ + + Example 3: Extract the minutes from a time column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... ("13:08:15",), + ... ("10:09:16",)], ['t']).withColumn("t", sf.col("t").cast("time")) + >>> df.select("*", sf.typeof('t'), sf.minute('t')).show() + +--------+---------+---------+ + | t|typeof(t)|minute(t)| + +--------+---------+---------+ + |13:08:15| time(6)| 8| + |10:09:16| time(6)| 9| + +--------+---------+---------+ """ - return _invoke_function_over_columns("make_date", year, month, day) + return _invoke_function_over_columns("minute", col) @_try_remote_functions -def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: +def second(col: "ColumnOrName") -> Column: """ - Returns the date that is `days` days after `start`. If `days` is a negative value - then these amount of days will be deducted from `start`. + Extract the seconds of a given date as integer. .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.1.0 + Added support for time type. + Parameters ---------- - start : :class:`~pyspark.sql.Column` or column name - date column to work on. - A column that evaluates to a date. - days : :class:`~pyspark.sql.Column` or column name or int - how many days after the given date to calculate. - Accepts negative value as well to calculate backwards in time. - A column that evaluates to an integer, short, or byte. + col : :class:`~pyspark.sql.Column` or column name + target date/time/timestamp column to work on. + A column that evaluates to a timestamp or time. Returns ------- :class:`~pyspark.sql.Column` - a date after/before given number of days. - Returns a column that evaluates to a date. + `seconds` part of the timestamp as integer. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.dateadd` - :meth:`pyspark.sql.functions.date_sub` - :meth:`pyspark.sql.functions.datediff` - :meth:`pyspark.sql.functions.date_diff` - :meth:`pyspark.sql.functions.timestamp_add` - :meth:`pyspark.sql.functions.add_months` + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.extract` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.date_part` Examples -------- + Example 1: Extract the seconds from a string column representing timestamp + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') - >>> df.select('*', sf.date_add(df.dt, 1)).show() - +----------+---+---------------+ - | dt| a|date_add(dt, 1)| - +----------+---+---------------+ - |2015-04-08| 2| 2015-04-09| - +----------+---+---------------+ + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.second('ts')).show() + +-------------------+----------+----------+ + | ts|typeof(ts)|second(ts)| + +-------------------+----------+----------+ + |2015-04-08 13:08:15| string| 15| + |2024-10-31 10:09:16| string| 16| + +-------------------+----------+----------+ - >>> df.select('*', sf.date_add('dt', 'a')).show() - +----------+---+---------------+ - | dt| a|date_add(dt, a)| - +----------+---+---------------+ - |2015-04-08| 2| 2015-04-10| - +----------+---+---------------+ + Example 2: Extract the seconds from a timestamp column - >>> df.select('*', sf.date_add('dt', sf.lit(-1))).show() - +----------+---+----------------+ - | dt| a|date_add(dt, -1)| - +----------+---+----------------+ - |2015-04-08| 2| 2015-04-07| - +----------+---+----------------+ + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.second('ts')).show() + +-------------------+----------+----------+ + | ts|typeof(ts)|second(ts)| + +-------------------+----------+----------+ + |2015-04-08 13:08:15| timestamp| 15| + |2024-10-31 10:09:16| timestamp| 16| + +-------------------+----------+----------+ + + Example 3: Extract the seconds from a time column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... ("13:08:15",), + ... ("10:09:16",)], ['t']).withColumn("t", sf.col("t").cast("time")) + >>> df.select("*", sf.typeof('t'), sf.second('t')).show() + +--------+---------+---------+ + | t|typeof(t)|second(t)| + +--------+---------+---------+ + |13:08:15| time(6)| 15| + |10:09:16| time(6)| 16| + +--------+---------+---------+ """ - days = _enum_to_value(days) - days = lit(days) if isinstance(days, int) else days - return _invoke_function_over_columns("date_add", start, days) + return _invoke_function_over_columns("second", col) @_try_remote_functions -def dateadd(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: +def weekofyear(col: "ColumnOrName") -> Column: """ - Returns the date that is `days` days after `start`. If `days` is a negative value - then these amount of days will be deducted from `start`. + Extract the week number of a given date as integer. + A week is considered to start on a Monday and week 1 is the first week with more than 3 days, + as defined by ISO 8601 - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - start : :class:`~pyspark.sql.Column` or column name - date column to work on. - A column that evaluates to a date. - days : :class:`~pyspark.sql.Column` or column name or int - how many days after the given date to calculate. - Accepts negative value as well to calculate backwards in time. - A column that evaluates to an integer, short, or byte. + col : :class:`~pyspark.sql.Column` or column name + target timestamp column to work on. + A column that evaluates to a date, timestamp or string. Returns ------- :class:`~pyspark.sql.Column` - a date after/before given number of days. - Returns a column that evaluates to a date. + `week` of the year for given date as integer. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.date_add` - :meth:`pyspark.sql.functions.date_sub` - :meth:`pyspark.sql.functions.datediff` - :meth:`pyspark.sql.functions.date_diff` - :meth:`pyspark.sql.functions.timestamp_add` - :meth:`pyspark.sql.functions.add_months` + :meth:`pyspark.sql.functions.weekday` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.dayofmonth` + :meth:`pyspark.sql.functions.dayofyear` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') - >>> df.select('*', sf.dateadd(df.dt, 1)).show() - +----------+---+---------------+ - | dt| a|date_add(dt, 1)| - +----------+---+---------------+ - |2015-04-08| 2| 2015-04-09| - +----------+---+---------------+ + Example 1: Extract the week of the year from a string column representing dates - >>> df.select('*', sf.dateadd('dt', 'a')).show() - +----------+---+---------------+ - | dt| a|date_add(dt, a)| - +----------+---+---------------+ - |2015-04-08| 2| 2015-04-10| - +----------+---+---------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.weekofyear('dt')).show() + +----------+----------+--------------+ + | dt|typeof(dt)|weekofyear(dt)| + +----------+----------+--------------+ + |2015-04-08| string| 15| + |2024-10-31| string| 44| + +----------+----------+--------------+ - >>> df.select('*', sf.dateadd('dt', sf.lit(-1))).show() - +----------+---+----------------+ - | dt| a|date_add(dt, -1)| - +----------+---+----------------+ - |2015-04-08| 2| 2015-04-07| - +----------+---+----------------+ + Example 2: Extract the week of the year from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.weekofyear('ts')).show() + +-------------------+----------+--------------+ + | ts|typeof(ts)|weekofyear(ts)| + +-------------------+----------+--------------+ + |2015-04-08 13:08:15| string| 15| + |2024-10-31 10:09:16| string| 44| + +-------------------+----------+--------------+ + + Example 3: Extract the week of the year from a date column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.weekofyear('dt')).show() + +----------+----------+--------------+ + | dt|typeof(dt)|weekofyear(dt)| + +----------+----------+--------------+ + |2015-04-08| date| 15| + |2024-10-31| date| 44| + +----------+----------+--------------+ + + Example 4: Extract the week of the year from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.weekofyear('ts')).show() + +-------------------+----------+--------------+ + | ts|typeof(ts)|weekofyear(ts)| + +-------------------+----------+--------------+ + |2015-04-08 13:08:15| timestamp| 15| + |2024-10-31 10:09:16| timestamp| 44| + +-------------------+----------+--------------+ """ - days = _enum_to_value(days) - days = lit(days) if isinstance(days, int) else days - return _invoke_function_over_columns("dateadd", start, days) + return _invoke_function_over_columns("weekofyear", col) @_try_remote_functions -def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: +def weekday(col: "ColumnOrName") -> Column: """ - Returns the date that is `days` days before `start`. If `days` is a negative value - then these amount of days will be added to `start`. - - .. versionadded:: 1.5.0 + Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - start : :class:`~pyspark.sql.Column` or column name - date column to work on. - A column that evaluates to a date. - days : :class:`~pyspark.sql.Column` or column name or int - how many days before the given date to calculate. - Accepts negative value as well to calculate forward in time. - A column that evaluates to an integer, short, or byte. + col : :class:`~pyspark.sql.Column` or column name + target date/timestamp column to work on. + A column that evaluates to a date, timestamp or string. Returns ------- :class:`~pyspark.sql.Column` - a date before/after given number of days. - Returns a column that evaluates to a date. + the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.dateadd` - :meth:`pyspark.sql.functions.date_add` - :meth:`pyspark.sql.functions.datediff` - :meth:`pyspark.sql.functions.date_diff` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.weekofyear` + :meth:`pyspark.sql.functions.dayofweek` + :meth:`pyspark.sql.functions.dayofyear` + :meth:`pyspark.sql.functions.dayofmonth` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') - >>> df.select('*', sf.date_sub(df.dt, 1)).show() - +----------+---+---------------+ - | dt| a|date_sub(dt, 1)| - +----------+---+---------------+ - |2015-04-08| 2| 2015-04-07| - +----------+---+---------------+ + Example 1: Extract the day of the week from a string column representing dates - >>> df.select('*', sf.date_sub('dt', 'a')).show() - +----------+---+---------------+ - | dt| a|date_sub(dt, a)| - +----------+---+---------------+ - |2015-04-08| 2| 2015-04-06| - +----------+---+---------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.weekday('dt')).show() + +----------+----------+-----------+ + | dt|typeof(dt)|weekday(dt)| + +----------+----------+-----------+ + |2015-04-08| string| 2| + |2024-10-31| string| 3| + +----------+----------+-----------+ - >>> df.select('*', sf.date_sub('dt', sf.lit(-1))).show() - +----------+---+----------------+ - | dt| a|date_sub(dt, -1)| - +----------+---+----------------+ - |2015-04-08| 2| 2015-04-09| - +----------+---+----------------+ + Example 2: Extract the day of the week from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.weekday('ts')).show() + +-------------------+----------+-----------+ + | ts|typeof(ts)|weekday(ts)| + +-------------------+----------+-----------+ + |2015-04-08 13:08:15| string| 2| + |2024-10-31 10:09:16| string| 3| + +-------------------+----------+-----------+ + + Example 3: Extract the day of the week from a date column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.weekday('dt')).show() + +----------+----------+-----------+ + | dt|typeof(dt)|weekday(dt)| + +----------+----------+-----------+ + |2015-04-08| date| 2| + |2024-10-31| date| 3| + +----------+----------+-----------+ + + Example 4: Extract the day of the week from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.weekday('ts')).show() + +-------------------+----------+-----------+ + | ts|typeof(ts)|weekday(ts)| + +-------------------+----------+-----------+ + |2015-04-08 13:08:15| timestamp| 2| + |2024-10-31 10:09:16| timestamp| 3| + +-------------------+----------+-----------+ """ - days = _enum_to_value(days) - days = lit(days) if isinstance(days, int) else days - return _invoke_function_over_columns("date_sub", start, days) + return _invoke_function_over_columns("weekday", col) @_try_remote_functions -def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column: +def monthname(col: "ColumnOrName") -> Column: """ - Returns the number of days from `start` to `end`. - - .. versionadded:: 1.5.0 + Returns the three-letter abbreviated month name from the given date. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - end : :class:`~pyspark.sql.Column` or column name - to date column to work on. - A column that evaluates to a date. - start : :class:`~pyspark.sql.Column` or column name - from date column to work on. - A column that evaluates to a date. + col : :class:`~pyspark.sql.Column` or column name + target date/timestamp column to work on. + A column that evaluates to a date, timestamp or string. Returns ------- :class:`~pyspark.sql.Column` - difference in days between two dates. - Returns a column that evaluates to an integer. + the three-letter abbreviation of month name for date/timestamp (Jan, Feb, Mar...) + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.dateadd` - :meth:`pyspark.sql.functions.date_add` - :meth:`pyspark.sql.functions.date_sub` - :meth:`pyspark.sql.functions.date_diff` - :meth:`pyspark.sql.functions.timestamp_diff` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.dayname` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2']) - >>> df.select('*', sf.datediff('d1', 'd2')).show() - +----------+----------+----------------+ - | d1| d2|datediff(d1, d2)| - +----------+----------+----------------+ - |2015-04-08|2015-05-10| -32| - +----------+----------+----------------+ + Example 1: Extract the month name from a string column representing dates - >>> df.select('*', sf.datediff(df.d2, df.d1)).show() - +----------+----------+----------------+ - | d1| d2|datediff(d2, d1)| - +----------+----------+----------------+ - |2015-04-08|2015-05-10| 32| - +----------+----------+----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.monthname('dt')).show() + +----------+----------+-------------+ + | dt|typeof(dt)|monthname(dt)| + +----------+----------+-------------+ + |2015-04-08| string| Apr| + |2024-10-31| string| Oct| + +----------+----------+-------------+ + + Example 2: Extract the month name from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.monthname('ts')).show() + +-------------------+----------+-------------+ + | ts|typeof(ts)|monthname(ts)| + +-------------------+----------+-------------+ + |2015-04-08 13:08:15| string| Apr| + |2024-10-31 10:09:16| string| Oct| + +-------------------+----------+-------------+ + + Example 3: Extract the month name from a date column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.monthname('dt')).show() + +----------+----------+-------------+ + | dt|typeof(dt)|monthname(dt)| + +----------+----------+-------------+ + |2015-04-08| date| Apr| + |2024-10-31| date| Oct| + +----------+----------+-------------+ + + Example 4: Extract the month name from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.monthname('ts')).show() + +-------------------+----------+-------------+ + | ts|typeof(ts)|monthname(ts)| + +-------------------+----------+-------------+ + |2015-04-08 13:08:15| timestamp| Apr| + |2024-10-31 10:09:16| timestamp| Oct| + +-------------------+----------+-------------+ """ - return _invoke_function_over_columns("datediff", end, start) + return _invoke_function_over_columns("monthname", col) @_try_remote_functions -def date_diff(end: "ColumnOrName", start: "ColumnOrName") -> Column: +def dayname(col: "ColumnOrName") -> Column: """ - Returns the number of days from `start` to `end`. + Date and Timestamp Function: Returns the three-letter abbreviated day name from the given date. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - end : :class:`~pyspark.sql.Column` or column name - to date column to work on. - A column that evaluates to a date. - start : :class:`~pyspark.sql.Column` or column name - from date column to work on. - A column that evaluates to a date. + col : :class:`~pyspark.sql.Column` or column name + target date/timestamp column to work on. + A column that evaluates to a date, timestamp or string. Returns ------- :class:`~pyspark.sql.Column` - difference in days between two dates. - Returns a column that evaluates to an integer. + the three-letter abbreviation of day name for date/timestamp (Mon, Tue, Wed...) + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.dateadd` - :meth:`pyspark.sql.functions.date_add` - :meth:`pyspark.sql.functions.date_sub` - :meth:`pyspark.sql.functions.datediff` - :meth:`pyspark.sql.functions.timestamp_diff` - :meth:`pyspark.sql.functions.time_diff` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.monthname` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2']) - >>> df.select('*', sf.date_diff('d1', 'd2')).show() - +----------+----------+-----------------+ - | d1| d2|date_diff(d1, d2)| - +----------+----------+-----------------+ - |2015-04-08|2015-05-10| -32| - +----------+----------+-----------------+ + Example 1: Extract the weekday name from a string column representing dates - >>> df.select('*', sf.date_diff(df.d2, df.d1)).show() - +----------+----------+-----------------+ - | d1| d2|date_diff(d2, d1)| - +----------+----------+-----------------+ - |2015-04-08|2015-05-10| 32| - +----------+----------+-----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08',), ('2024-10-31',)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.dayname('dt')).show() + +----------+----------+-----------+ + | dt|typeof(dt)|dayname(dt)| + +----------+----------+-----------+ + |2015-04-08| string| Wed| + |2024-10-31| string| Thu| + +----------+----------+-----------+ + + Example 2: Extract the weekday name from a string column representing timestamp + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08 13:08:15',), ('2024-10-31 10:09:16',)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.dayname('ts')).show() + +-------------------+----------+-----------+ + | ts|typeof(ts)|dayname(ts)| + +-------------------+----------+-----------+ + |2015-04-08 13:08:15| string| Wed| + |2024-10-31 10:09:16| string| Thu| + +-------------------+----------+-----------+ + + Example 3: Extract the weekday name from a date column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.date(2015, 4, 8),), + ... (datetime.date(2024, 10, 31),)], ['dt']) + >>> df.select("*", sf.typeof('dt'), sf.dayname('dt')).show() + +----------+----------+-----------+ + | dt|typeof(dt)|dayname(dt)| + +----------+----------+-----------+ + |2015-04-08| date| Wed| + |2024-10-31| date| Thu| + +----------+----------+-----------+ + + Example 4: Extract the weekday name from a timestamp column + + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (datetime.datetime(2015, 4, 8, 13, 8, 15),), + ... (datetime.datetime(2024, 10, 31, 10, 9, 16),)], ['ts']) + >>> df.select("*", sf.typeof('ts'), sf.dayname('ts')).show() + +-------------------+----------+-----------+ + | ts|typeof(ts)|dayname(ts)| + +-------------------+----------+-----------+ + |2015-04-08 13:08:15| timestamp| Wed| + |2024-10-31 10:09:16| timestamp| Thu| + +-------------------+----------+-----------+ """ - return _invoke_function_over_columns("date_diff", end, start) + return _invoke_function_over_columns("dayname", col) @_try_remote_functions -def date_from_unix_date(days: "ColumnOrName") -> Column: +def extract(field: Column, source: "ColumnOrName") -> Column: """ - Create date from the number of `days` since 1970-01-01. + Extracts a part of the date/timestamp or interval source. .. versionadded:: 3.5.0 Parameters ---------- - days : :class:`~pyspark.sql.Column` or column name - the target column to work on. - A column that evaluates to an integer. + field : :class:`~pyspark.sql.Column` + selects which part of the source should be extracted. + source : :class:`~pyspark.sql.Column` or column name + a date, time, timestamp, or interval column from where `field` should be extracted. Returns ------- :class:`~pyspark.sql.Column` - the date from the number of days since 1970-01-01. - Returns a column that evaluates to a date. + a part of the date/timestamp or interval source. + Returns a column whose type depends on the field to extract, e.g. an integer + for ``YEAR`` and a decimal for ``SECOND``. See Also -------- - :meth:`pyspark.sql.functions.from_unixtime` - :meth:`pyspark.sql.functions.unix_date` + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.date_part` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(4).select('*', sf.date_from_unix_date('id')).show() - +---+-----------------------+ - | id|date_from_unix_date(id)| - +---+-----------------------+ - | 0| 1970-01-01| - | 1| 1970-01-02| - | 2| 1970-01-03| - | 3| 1970-01-04| - +---+-----------------------+ - """ - return _invoke_function_over_columns("date_from_unix_date", days) - - -@_try_remote_functions -def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Column: - """ - Returns the date that is `months` months after `start`. If `months` is a negative value - then these amount of months will be deducted from the `start`. + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts']) + >>> df.select( + ... '*', + ... sf.extract(sf.lit('YEAR'), 'ts').alias('year'), + ... sf.extract(sf.lit('month'), 'ts').alias('month'), + ... sf.extract(sf.lit('WEEK'), 'ts').alias('week'), + ... sf.extract(sf.lit('D'), df.ts).alias('day'), + ... sf.extract(sf.lit('M'), df.ts).alias('minute'), + ... sf.extract(sf.lit('S'), df.ts).alias('second') + ... ).show() + +-------------------+----+-----+----+---+------+---------+ + | ts|year|month|week|day|minute| second| + +-------------------+----+-----+----+---+------+---------+ + |2015-04-08 13:08:15|2015| 4| 15| 8| 8|15.000000| + +-------------------+----+-----+----+---+------+---------+ + """ + return _invoke_function_over_columns("extract", field, source) - .. versionadded:: 1.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def date_part(field: Column, source: "ColumnOrName") -> Column: + """ + Extracts a part of the date/timestamp or interval source. + + .. versionadded:: 3.5.0 Parameters ---------- - start : :class:`~pyspark.sql.Column` or column name - date column to work on. - A column that evaluates to a date. - months : :class:`~pyspark.sql.Column` or column name or int - how many months after the given date to calculate. - Accepts negative value as well to calculate backwards. - A column that evaluates to an integer. + field : :class:`~pyspark.sql.Column` + selects which part of the source should be extracted, and supported string values + are as same as the fields of the equivalent function `extract`. + source : :class:`~pyspark.sql.Column` or column name + a date, time, timestamp, or interval column from where `field` should be extracted. Returns ------- :class:`~pyspark.sql.Column` - a date after/before given number of months. - Returns a column that evaluates to a date. + a part of the date/timestamp or interval source. + Returns a column whose type depends on the field to extract, e.g. an integer + for ``YEAR`` and a decimal for ``SECOND``. See Also -------- - :meth:`pyspark.sql.functions.dateadd` - :meth:`pyspark.sql.functions.date_add` + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.datepart` + :meth:`pyspark.sql.functions.extract` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') - >>> df.select('*', sf.add_months(df.dt, 1)).show() - +----------+---+-----------------+ - | dt| a|add_months(dt, 1)| - +----------+---+-----------------+ - |2015-04-08| 2| 2015-05-08| - +----------+---+-----------------+ - - >>> df.select('*', sf.add_months('dt', 'a')).show() - +----------+---+-----------------+ - | dt| a|add_months(dt, a)| - +----------+---+-----------------+ - |2015-04-08| 2| 2015-06-08| - +----------+---+-----------------+ - - >>> df.select('*', sf.add_months('dt', sf.lit(-1))).show() - +----------+---+------------------+ - | dt| a|add_months(dt, -1)| - +----------+---+------------------+ - |2015-04-08| 2| 2015-03-08| - +----------+---+------------------+ + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts']) + >>> df.select( + ... '*', + ... sf.date_part(sf.lit('YEAR'), 'ts').alias('year'), + ... sf.date_part(sf.lit('month'), 'ts').alias('month'), + ... sf.date_part(sf.lit('WEEK'), 'ts').alias('week'), + ... sf.date_part(sf.lit('D'), df.ts).alias('day'), + ... sf.date_part(sf.lit('M'), df.ts).alias('minute'), + ... sf.date_part(sf.lit('S'), df.ts).alias('second') + ... ).show() + +-------------------+----+-----+----+---+------+---------+ + | ts|year|month|week|day|minute| second| + +-------------------+----+-----+----+---+------+---------+ + |2015-04-08 13:08:15|2015| 4| 15| 8| 8|15.000000| + +-------------------+----+-----+----+---+------+---------+ """ - months = _enum_to_value(months) - months = lit(months) if isinstance(months, int) else months - return _invoke_function_over_columns("add_months", start, months) + return _invoke_function_over_columns("date_part", field, source) @_try_remote_functions -def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool = True) -> Column: +def datepart(field: Column, source: "ColumnOrName") -> Column: """ - Returns number of months between dates date1 and date2. - If date1 is later than date2, then the result is positive. - A whole number is returned if both inputs have the same day of month or both are the last day - of their respective months. Otherwise, the difference is calculated assuming 31 days per month. - The result is rounded off to 8 digits unless `roundOff` is set to `False`. - - .. versionadded:: 1.5.0 + Extracts a part of the date/timestamp or interval source. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - date1 : :class:`~pyspark.sql.Column` or column name - first date column. - A column that evaluates to a timestamp. - date2 : :class:`~pyspark.sql.Column` or column name - second date column. - A column that evaluates to a timestamp. - roundOff : bool, optional - whether to round (to 8 digits) the final value or not (default: True). - A column that evaluates to a boolean. + field : :class:`~pyspark.sql.Column` + selects which part of the source should be extracted, and supported string values + are as same as the fields of the equivalent function `extract`. + source : :class:`~pyspark.sql.Column` or column name + a date, time, timestamp, or interval column from where `field` should be extracted. Returns ------- :class:`~pyspark.sql.Column` - number of months between two dates. - Returns a column that evaluates to a double. + a part of the date/timestamp or interval source. + Returns a column whose type depends on the field to extract, e.g. an integer + for ``YEAR`` and a decimal for ``SECOND``. - Examples + See Also -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['d1', 'd2']) - >>> df.select('*', sf.months_between(df.d1, df.d2)).show() - +-------------------+----------+----------------------------+ - | d1| d2|months_between(d1, d2, true)| - +-------------------+----------+----------------------------+ - |1997-02-28 10:30:00|1996-10-30| 3.94959677| - +-------------------+----------+----------------------------+ - - >>> df.select('*', sf.months_between('d2', 'd1')).show() - +-------------------+----------+----------------------------+ - | d1| d2|months_between(d2, d1, true)| - +-------------------+----------+----------------------------+ - |1997-02-28 10:30:00|1996-10-30| -3.94959677| - +-------------------+----------+----------------------------+ + :meth:`pyspark.sql.functions.year` + :meth:`pyspark.sql.functions.quarter` + :meth:`pyspark.sql.functions.month` + :meth:`pyspark.sql.functions.day` + :meth:`pyspark.sql.functions.hour` + :meth:`pyspark.sql.functions.minute` + :meth:`pyspark.sql.functions.second` + :meth:`pyspark.sql.functions.date_part` + :meth:`pyspark.sql.functions.extract` - >>> df.select('*', sf.months_between('d1', df.d2, False)).show() - +-------------------+----------+-----------------------------+ - | d1| d2|months_between(d1, d2, false)| - +-------------------+----------+-----------------------------+ - |1997-02-28 10:30:00|1996-10-30| 3.9495967741935...| - +-------------------+----------+-----------------------------+ + Examples + -------- + >>> import datetime + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(datetime.datetime(2015, 4, 8, 13, 8, 15),)], ['ts']) + >>> df.select( + ... '*', + ... sf.datepart(sf.lit('YEAR'), 'ts').alias('year'), + ... sf.datepart(sf.lit('month'), 'ts').alias('month'), + ... sf.datepart(sf.lit('WEEK'), 'ts').alias('week'), + ... sf.datepart(sf.lit('D'), df.ts).alias('day'), + ... sf.datepart(sf.lit('M'), df.ts).alias('minute'), + ... sf.datepart(sf.lit('S'), df.ts).alias('second') + ... ).show() + +-------------------+----+-----+----+---+------+---------+ + | ts|year|month|week|day|minute| second| + +-------------------+----+-----+----+---+------+---------+ + |2015-04-08 13:08:15|2015| 4| 15| 8| 8|15.000000| + +-------------------+----+-----+----+---+------+---------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "months_between", _to_java_column(date1), _to_java_column(date2), _enum_to_value(roundOff) - ) + return _invoke_function_over_columns("datepart", field, source) @_try_remote_functions -def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: - """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.DateType` - using the optionally specified format. Specify formats according to `datetime pattern`_. - By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format - is omitted. Equivalent to ``col.cast("date")``. - - .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html +def make_date(year: "ColumnOrName", month: "ColumnOrName", day: "ColumnOrName") -> Column: + """ + Returns a column with a date built from the year, month and day columns. - .. versionadded:: 2.2.0 + .. versionadded:: 3.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to convert. - A column that evaluates to a string, date, or timestamp. - format: literal string, optional - format to use to convert date values. - A column that evaluates to a string. + year : :class:`~pyspark.sql.Column` or column name + The year to build the date. + A column that evaluates to an integer. + month : :class:`~pyspark.sql.Column` or column name + The month to build the date. + A column that evaluates to an integer. + day : :class:`~pyspark.sql.Column` or column name + The day to build the date. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - date value as :class:`pyspark.sql.types.DateType` type. + a date built from given parts. Returns a column that evaluates to a date. See Also -------- - :meth:`pyspark.sql.functions.to_timestamp` - :meth:`pyspark.sql.functions.to_timestamp_ltz` - :meth:`pyspark.sql.functions.to_timestamp_ntz` - :meth:`pyspark.sql.functions.to_utc_timestamp` - :meth:`pyspark.sql.functions.try_to_timestamp` - :meth:`pyspark.sql.functions.date_format` - :meth:`pyspark.sql.functions.try_to_date` + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['ts']) - >>> df.select('*', sf.to_date(df.ts)).show() - +-------------------+-----------+ - | ts|to_date(ts)| - +-------------------+-----------+ - |1997-02-28 10:30:00| 1997-02-28| - +-------------------+-----------+ - - >>> df.select('*', sf.to_date('ts', 'yyyy-MM-dd HH:mm:ss')).show() - +-------------------+--------------------------------+ - | ts|to_date(ts, yyyy-MM-dd HH:mm:ss)| - +-------------------+--------------------------------+ - |1997-02-28 10:30:00| 1997-02-28| - +-------------------+--------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(2020, 6, 26)], ['Y', 'M', 'D']) + >>> df.select('*', sf.make_date(df.Y, 'M', df.D)).show() + +----+---+---+------------------+ + | Y| M| D|make_date(Y, M, D)| + +----+---+---+------------------+ + |2020| 6| 26| 2020-06-26| + +----+---+---+------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if format is None: - return _invoke_function_over_columns("to_date", col) - else: - return _invoke_function("to_date", _to_java_column(col), _enum_to_value(format)) + return _invoke_function_over_columns("make_date", year, month, day) @_try_remote_functions -def try_to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: - """This is a special version of `to_date` that performs the same operation, but returns a - NULL value instead of raising an error if date cannot be created. +def date_add(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: + """ + Returns the date that is `days` days after `start`. If `days` is a negative value + then these amount of days will be deducted from `start`. - .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html + .. versionadded:: 1.5.0 - .. versionadded:: 4.1.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to convert. - A column that evaluates to a string, date, or timestamp. - format: literal string, optional - format to use to convert date values. - A column that evaluates to a string. + start : :class:`~pyspark.sql.Column` or column name + date column to work on. + A column that evaluates to a date. + days : :class:`~pyspark.sql.Column` or column name or int + how many days after the given date to calculate. + Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` - date value as :class:`pyspark.sql.types.DateType` type. + a date after/before given number of days. Returns a column that evaluates to a date. See Also -------- - :meth:`pyspark.sql.functions.to_timestamp` - :meth:`pyspark.sql.functions.to_timestamp_ltz` - :meth:`pyspark.sql.functions.to_timestamp_ntz` - :meth:`pyspark.sql.functions.to_utc_timestamp` - :meth:`pyspark.sql.functions.try_to_timestamp` - :meth:`pyspark.sql.functions.date_format` - :meth:`pyspark.sql.functions.to_date` + :meth:`pyspark.sql.functions.dateadd` + :meth:`pyspark.sql.functions.date_sub` + :meth:`pyspark.sql.functions.datediff` + :meth:`pyspark.sql.functions.date_diff` + :meth:`pyspark.sql.functions.timestamp_add` + :meth:`pyspark.sql.functions.add_months` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('1997-02-28',)], ['ts']) - >>> df.select('*', sf.try_to_date(df.ts)).show() - +----------+---------------+ - | ts|try_to_date(ts)| - +----------+---------------+ - |1997-02-28| 1997-02-28| - +----------+---------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') + >>> df.select('*', sf.date_add(df.dt, 1)).show() + +----------+---+---------------+ + | dt| a|date_add(dt, 1)| + +----------+---+---------------+ + |2015-04-08| 2| 2015-04-09| + +----------+---+---------------+ - >>> df.select('*', sf.try_to_date('ts', 'yyyy-MM-dd')).show() - +----------+---------------------------+ - | ts|try_to_date(ts, yyyy-MM-dd)| - +----------+---------------------------+ - |1997-02-28| 1997-02-28| - +----------+---------------------------+ + >>> df.select('*', sf.date_add('dt', 'a')).show() + +----------+---+---------------+ + | dt| a|date_add(dt, a)| + +----------+---+---------------+ + |2015-04-08| 2| 2015-04-10| + +----------+---+---------------+ - >>> df = spark.createDataFrame([('foo',)], ['ts']) - >>> df.select(sf.try_to_date(df.ts)).show() - +---------------+ - |try_to_date(ts)| - +---------------+ - | NULL| - +---------------+ + >>> df.select('*', sf.date_add('dt', sf.lit(-1))).show() + +----------+---+----------------+ + | dt| a|date_add(dt, -1)| + +----------+---+----------------+ + |2015-04-08| 2| 2015-04-07| + +----------+---+----------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if format is None: - return _invoke_function_over_columns("try_to_date", col) - else: - return _invoke_function("try_to_date", _to_java_column(col), _enum_to_value(format)) + days = _enum_to_value(days) + days = lit(days) if isinstance(days, int) else days + return _invoke_function_over_columns("date_add", start, days) @_try_remote_functions -def unix_date(col: "ColumnOrName") -> Column: - """Returns the number of days since 1970-01-01. +def dateadd(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: + """ + Returns the date that is `days` days after `start`. If `days` is a negative value + then these amount of days will be deducted from `start`. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to convert. + start : :class:`~pyspark.sql.Column` or column name + date column to work on. A column that evaluates to a date. + days : :class:`~pyspark.sql.Column` or column name or int + how many days after the given date to calculate. + Accepts negative value as well to calculate backwards in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` - the number of days since 1970-01-01. - Returns a column that evaluates to an integer. + a date after/before given number of days. + Returns a column that evaluates to a date. See Also -------- - :meth:`pyspark.sql.functions.date_from_unix_date` - :meth:`pyspark.sql.functions.unix_seconds` - :meth:`pyspark.sql.functions.unix_millis` - :meth:`pyspark.sql.functions.unix_micros` + :meth:`pyspark.sql.functions.date_add` + :meth:`pyspark.sql.functions.date_sub` + :meth:`pyspark.sql.functions.datediff` + :meth:`pyspark.sql.functions.date_diff` + :meth:`pyspark.sql.functions.timestamp_add` + :meth:`pyspark.sql.functions.add_months` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('1970-01-02',), ('2022-01-02',)], ['dt']) - >>> df.select('*', sf.unix_date(sf.to_date('dt'))).show() - +----------+----------------------+ - | dt|unix_date(to_date(dt))| - +----------+----------------------+ - |1970-01-02| 1| - |2022-01-02| 18994| - +----------+----------------------+ + >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') + >>> df.select('*', sf.dateadd(df.dt, 1)).show() + +----------+---+---------------+ + | dt| a|date_add(dt, 1)| + +----------+---+---------------+ + |2015-04-08| 2| 2015-04-09| + +----------+---+---------------+ - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> df.select('*', sf.dateadd('dt', 'a')).show() + +----------+---+---------------+ + | dt| a|date_add(dt, a)| + +----------+---+---------------+ + |2015-04-08| 2| 2015-04-10| + +----------+---+---------------+ + + >>> df.select('*', sf.dateadd('dt', sf.lit(-1))).show() + +----------+---+----------------+ + | dt| a|date_add(dt, -1)| + +----------+---+----------------+ + |2015-04-08| 2| 2015-04-07| + +----------+---+----------------+ """ - return _invoke_function_over_columns("unix_date", col) + days = _enum_to_value(days) + days = lit(days) if isinstance(days, int) else days + return _invoke_function_over_columns("dateadd", start, days) @_try_remote_functions -def unix_micros(col: "ColumnOrName") -> Column: - """Returns the number of microseconds since 1970-01-01 00:00:00 UTC. - Truncates higher levels of precision. +def date_sub(start: "ColumnOrName", days: Union["ColumnOrName", int]) -> Column: + """ + Returns the date that is `days` days before `start`. If `days` is a negative value + then these amount of days will be added to `start`. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to convert. - A column that evaluates to a timestamp. + start : :class:`~pyspark.sql.Column` or column name + date column to work on. + A column that evaluates to a date. + days : :class:`~pyspark.sql.Column` or column name or int + how many days before the given date to calculate. + Accepts negative value as well to calculate forward in time. + A column that evaluates to an integer, short, or byte. Returns ------- :class:`~pyspark.sql.Column` - the number of microseconds since 1970-01-01 00:00:00 UTC. - Returns a column that evaluates to a long. + a date before/after given number of days. + Returns a column that evaluates to a date. See Also -------- - :meth:`pyspark.sql.functions.unix_date` - :meth:`pyspark.sql.functions.unix_seconds` - :meth:`pyspark.sql.functions.unix_millis` - :meth:`pyspark.sql.functions.unix_nanos` - :meth:`pyspark.sql.functions.timestamp_micros` + :meth:`pyspark.sql.functions.dateadd` + :meth:`pyspark.sql.functions.date_add` + :meth:`pyspark.sql.functions.datediff` + :meth:`pyspark.sql.functions.date_diff` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-07-22 10:00:00',), ('2022-10-09 11:12:13',)], ['ts']) - >>> df.select('*', sf.unix_micros(sf.to_timestamp('ts'))).show() - +-------------------+-----------------------------+ - | ts|unix_micros(to_timestamp(ts))| - +-------------------+-----------------------------+ - |2015-07-22 10:00:00| 1437584400000000| - |2022-10-09 11:12:13| 1665339133000000| - +-------------------+-----------------------------+ + >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') + >>> df.select('*', sf.date_sub(df.dt, 1)).show() + +----------+---+---------------+ + | dt| a|date_sub(dt, 1)| + +----------+---+---------------+ + |2015-04-08| 2| 2015-04-07| + +----------+---+---------------+ - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> df.select('*', sf.date_sub('dt', 'a')).show() + +----------+---+---------------+ + | dt| a|date_sub(dt, a)| + +----------+---+---------------+ + |2015-04-08| 2| 2015-04-06| + +----------+---+---------------+ + + >>> df.select('*', sf.date_sub('dt', sf.lit(-1))).show() + +----------+---+----------------+ + | dt| a|date_sub(dt, -1)| + +----------+---+----------------+ + |2015-04-08| 2| 2015-04-09| + +----------+---+----------------+ """ - return _invoke_function_over_columns("unix_micros", col) + days = _enum_to_value(days) + days = lit(days) if isinstance(days, int) else days + return _invoke_function_over_columns("date_sub", start, days) @_try_remote_functions -def unix_millis(col: "ColumnOrName") -> Column: - """Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. - Truncates higher levels of precision. +def datediff(end: "ColumnOrName", start: "ColumnOrName") -> Column: + """ + Returns the number of days from `start` to `end`. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to convert. - A column that evaluates to a timestamp. + end : :class:`~pyspark.sql.Column` or column name + to date column to work on. + A column that evaluates to a date. + start : :class:`~pyspark.sql.Column` or column name + from date column to work on. + A column that evaluates to a date. Returns ------- :class:`~pyspark.sql.Column` - the number of milliseconds since 1970-01-01 00:00:00 UTC. - Returns a column that evaluates to a long. + difference in days between two dates. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.unix_date` - :meth:`pyspark.sql.functions.unix_seconds` - :meth:`pyspark.sql.functions.unix_micros` - :meth:`pyspark.sql.functions.unix_nanos` - :meth:`pyspark.sql.functions.timestamp_millis` - - Examples + :meth:`pyspark.sql.functions.dateadd` + :meth:`pyspark.sql.functions.date_add` + :meth:`pyspark.sql.functions.date_sub` + :meth:`pyspark.sql.functions.date_diff` + :meth:`pyspark.sql.functions.timestamp_diff` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2']) + >>> df.select('*', sf.datediff('d1', 'd2')).show() + +----------+----------+----------------+ + | d1| d2|datediff(d1, d2)| + +----------+----------+----------------+ + |2015-04-08|2015-05-10| -32| + +----------+----------+----------------+ + + >>> df.select('*', sf.datediff(df.d2, df.d1)).show() + +----------+----------+----------------+ + | d1| d2|datediff(d2, d1)| + +----------+----------+----------------+ + |2015-04-08|2015-05-10| 32| + +----------+----------+----------------+ + """ + return _invoke_function_over_columns("datediff", end, start) + + +@_try_remote_functions +def date_diff(end: "ColumnOrName", start: "ColumnOrName") -> Column: + """ + Returns the number of days from `start` to `end`. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + end : :class:`~pyspark.sql.Column` or column name + to date column to work on. + A column that evaluates to a date. + start : :class:`~pyspark.sql.Column` or column name + from date column to work on. + A column that evaluates to a date. + + Returns + ------- + :class:`~pyspark.sql.Column` + difference in days between two dates. + Returns a column that evaluates to an integer. + + See Also + -------- + :meth:`pyspark.sql.functions.dateadd` + :meth:`pyspark.sql.functions.date_add` + :meth:`pyspark.sql.functions.date_sub` + :meth:`pyspark.sql.functions.datediff` + :meth:`pyspark.sql.functions.timestamp_diff` + :meth:`pyspark.sql.functions.time_diff` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('2015-04-08','2015-05-10')], ['d1', 'd2']) + >>> df.select('*', sf.date_diff('d1', 'd2')).show() + +----------+----------+-----------------+ + | d1| d2|date_diff(d1, d2)| + +----------+----------+-----------------+ + |2015-04-08|2015-05-10| -32| + +----------+----------+-----------------+ + + >>> df.select('*', sf.date_diff(df.d2, df.d1)).show() + +----------+----------+-----------------+ + | d1| d2|date_diff(d2, d1)| + +----------+----------+-----------------+ + |2015-04-08|2015-05-10| 32| + +----------+----------+-----------------+ + """ + return _invoke_function_over_columns("date_diff", end, start) + + +@_try_remote_functions +def date_from_unix_date(days: "ColumnOrName") -> Column: + """ + Create date from the number of `days` since 1970-01-01. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + days : :class:`~pyspark.sql.Column` or column name + the target column to work on. + A column that evaluates to an integer. + + Returns + ------- + :class:`~pyspark.sql.Column` + the date from the number of days since 1970-01-01. + Returns a column that evaluates to a date. + + See Also + -------- + :meth:`pyspark.sql.functions.from_unixtime` + :meth:`pyspark.sql.functions.unix_date` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(4).select('*', sf.date_from_unix_date('id')).show() + +---+-----------------------+ + | id|date_from_unix_date(id)| + +---+-----------------------+ + | 0| 1970-01-01| + | 1| 1970-01-02| + | 2| 1970-01-03| + | 3| 1970-01-04| + +---+-----------------------+ + """ + return _invoke_function_over_columns("date_from_unix_date", days) + + +@_try_remote_functions +def add_months(start: "ColumnOrName", months: Union["ColumnOrName", int]) -> Column: + """ + Returns the date that is `months` months after `start`. If `months` is a negative value + then these amount of months will be deducted from the `start`. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + start : :class:`~pyspark.sql.Column` or column name + date column to work on. + A column that evaluates to a date. + months : :class:`~pyspark.sql.Column` or column name or int + how many months after the given date to calculate. + Accepts negative value as well to calculate backwards. + A column that evaluates to an integer. + + Returns + ------- + :class:`~pyspark.sql.Column` + a date after/before given number of months. + Returns a column that evaluates to a date. + + See Also + -------- + :meth:`pyspark.sql.functions.dateadd` + :meth:`pyspark.sql.functions.date_add` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('2015-04-08', 2,)], 'struct') + >>> df.select('*', sf.add_months(df.dt, 1)).show() + +----------+---+-----------------+ + | dt| a|add_months(dt, 1)| + +----------+---+-----------------+ + |2015-04-08| 2| 2015-05-08| + +----------+---+-----------------+ + + >>> df.select('*', sf.add_months('dt', 'a')).show() + +----------+---+-----------------+ + | dt| a|add_months(dt, a)| + +----------+---+-----------------+ + |2015-04-08| 2| 2015-06-08| + +----------+---+-----------------+ + + >>> df.select('*', sf.add_months('dt', sf.lit(-1))).show() + +----------+---+------------------+ + | dt| a|add_months(dt, -1)| + +----------+---+------------------+ + |2015-04-08| 2| 2015-03-08| + +----------+---+------------------+ + """ + months = _enum_to_value(months) + months = lit(months) if isinstance(months, int) else months + return _invoke_function_over_columns("add_months", start, months) + + +@_try_remote_functions +def months_between(date1: "ColumnOrName", date2: "ColumnOrName", roundOff: bool = True) -> Column: + """ + Returns number of months between dates date1 and date2. + If date1 is later than date2, then the result is positive. + A whole number is returned if both inputs have the same day of month or both are the last day + of their respective months. Otherwise, the difference is calculated assuming 31 days per month. + The result is rounded off to 8 digits unless `roundOff` is set to `False`. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + date1 : :class:`~pyspark.sql.Column` or column name + first date column. + A column that evaluates to a timestamp. + date2 : :class:`~pyspark.sql.Column` or column name + second date column. + A column that evaluates to a timestamp. + roundOff : bool, optional + whether to round (to 8 digits) the final value or not (default: True). + A column that evaluates to a boolean. + + Returns + ------- + :class:`~pyspark.sql.Column` + number of months between two dates. + Returns a column that evaluates to a double. + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('1997-02-28 10:30:00', '1996-10-30')], ['d1', 'd2']) + >>> df.select('*', sf.months_between(df.d1, df.d2)).show() + +-------------------+----------+----------------------------+ + | d1| d2|months_between(d1, d2, true)| + +-------------------+----------+----------------------------+ + |1997-02-28 10:30:00|1996-10-30| 3.94959677| + +-------------------+----------+----------------------------+ + + >>> df.select('*', sf.months_between('d2', 'd1')).show() + +-------------------+----------+----------------------------+ + | d1| d2|months_between(d2, d1, true)| + +-------------------+----------+----------------------------+ + |1997-02-28 10:30:00|1996-10-30| -3.94959677| + +-------------------+----------+----------------------------+ + + >>> df.select('*', sf.months_between('d1', df.d2, False)).show() + +-------------------+----------+-----------------------------+ + | d1| d2|months_between(d1, d2, false)| + +-------------------+----------+-----------------------------+ + |1997-02-28 10:30:00|1996-10-30| 3.9495967741935...| + +-------------------+----------+-----------------------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "months_between", _to_java_column(date1), _to_java_column(date2), _enum_to_value(roundOff) + ) + + +@_try_remote_functions +def to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: + """Converts a :class:`~pyspark.sql.Column` into :class:`pyspark.sql.types.DateType` + using the optionally specified format. Specify formats according to `datetime pattern`_. + By default, it follows casting rules to :class:`pyspark.sql.types.DateType` if the format + is omitted. Equivalent to ``col.cast("date")``. + + .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html + + .. versionadded:: 2.2.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input column of values to convert. + A column that evaluates to a string, date, or timestamp. + format: literal string, optional + format to use to convert date values. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. + + See Also + -------- + :meth:`pyspark.sql.functions.to_timestamp` + :meth:`pyspark.sql.functions.to_timestamp_ltz` + :meth:`pyspark.sql.functions.to_timestamp_ntz` + :meth:`pyspark.sql.functions.to_utc_timestamp` + :meth:`pyspark.sql.functions.try_to_timestamp` + :meth:`pyspark.sql.functions.date_format` + :meth:`pyspark.sql.functions.try_to_date` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('1997-02-28 10:30:00',)], ['ts']) + >>> df.select('*', sf.to_date(df.ts)).show() + +-------------------+-----------+ + | ts|to_date(ts)| + +-------------------+-----------+ + |1997-02-28 10:30:00| 1997-02-28| + +-------------------+-----------+ + + >>> df.select('*', sf.to_date('ts', 'yyyy-MM-dd HH:mm:ss')).show() + +-------------------+--------------------------------+ + | ts|to_date(ts, yyyy-MM-dd HH:mm:ss)| + +-------------------+--------------------------------+ + |1997-02-28 10:30:00| 1997-02-28| + +-------------------+--------------------------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + if format is None: + return _invoke_function_over_columns("to_date", col) + else: + return _invoke_function("to_date", _to_java_column(col), _enum_to_value(format)) + + +@_try_remote_functions +def try_to_date(col: "ColumnOrName", format: Optional[str] = None) -> Column: + """This is a special version of `to_date` that performs the same operation, but returns a + NULL value instead of raising an error if date cannot be created. + + .. _datetime pattern: https://spark.apache.org/docs/latest/sql-ref-datetime-pattern.html + + .. versionadded:: 4.1.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input column of values to convert. + A column that evaluates to a string, date, or timestamp. + format: literal string, optional + format to use to convert date values. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + date value as :class:`pyspark.sql.types.DateType` type. + Returns a column that evaluates to a date. + + See Also + -------- + :meth:`pyspark.sql.functions.to_timestamp` + :meth:`pyspark.sql.functions.to_timestamp_ltz` + :meth:`pyspark.sql.functions.to_timestamp_ntz` + :meth:`pyspark.sql.functions.to_utc_timestamp` + :meth:`pyspark.sql.functions.try_to_timestamp` + :meth:`pyspark.sql.functions.date_format` + :meth:`pyspark.sql.functions.to_date` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('1997-02-28',)], ['ts']) + >>> df.select('*', sf.try_to_date(df.ts)).show() + +----------+---------------+ + | ts|try_to_date(ts)| + +----------+---------------+ + |1997-02-28| 1997-02-28| + +----------+---------------+ + + >>> df.select('*', sf.try_to_date('ts', 'yyyy-MM-dd')).show() + +----------+---------------------------+ + | ts|try_to_date(ts, yyyy-MM-dd)| + +----------+---------------------------+ + |1997-02-28| 1997-02-28| + +----------+---------------------------+ + + >>> df = spark.createDataFrame([('foo',)], ['ts']) + >>> df.select(sf.try_to_date(df.ts)).show() + +---------------+ + |try_to_date(ts)| + +---------------+ + | NULL| + +---------------+ + """ + from pyspark.sql.classic.column import _to_java_column + + if format is None: + return _invoke_function_over_columns("try_to_date", col) + else: + return _invoke_function("try_to_date", _to_java_column(col), _enum_to_value(format)) + + +@_try_remote_functions +def unix_date(col: "ColumnOrName") -> Column: + """Returns the number of days since 1970-01-01. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input column of values to convert. + A column that evaluates to a date. + + Returns + ------- + :class:`~pyspark.sql.Column` + the number of days since 1970-01-01. + Returns a column that evaluates to an integer. + + See Also + -------- + :meth:`pyspark.sql.functions.date_from_unix_date` + :meth:`pyspark.sql.functions.unix_seconds` + :meth:`pyspark.sql.functions.unix_millis` + :meth:`pyspark.sql.functions.unix_micros` + + Examples + -------- + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('1970-01-02',), ('2022-01-02',)], ['dt']) + >>> df.select('*', sf.unix_date(sf.to_date('dt'))).show() + +----------+----------------------+ + | dt|unix_date(to_date(dt))| + +----------+----------------------+ + |1970-01-02| 1| + |2022-01-02| 18994| + +----------+----------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") + """ + return _invoke_function_over_columns("unix_date", col) + + +@_try_remote_functions +def unix_micros(col: "ColumnOrName") -> Column: + """Returns the number of microseconds since 1970-01-01 00:00:00 UTC. + Truncates higher levels of precision. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input column of values to convert. + A column that evaluates to a timestamp. + + Returns + ------- + :class:`~pyspark.sql.Column` + the number of microseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. + + See Also + -------- + :meth:`pyspark.sql.functions.unix_date` + :meth:`pyspark.sql.functions.unix_seconds` + :meth:`pyspark.sql.functions.unix_millis` + :meth:`pyspark.sql.functions.unix_nanos` + :meth:`pyspark.sql.functions.timestamp_micros` + + Examples + -------- + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('2015-07-22 10:00:00',), ('2022-10-09 11:12:13',)], ['ts']) + >>> df.select('*', sf.unix_micros(sf.to_timestamp('ts'))).show() + +-------------------+-----------------------------+ + | ts|unix_micros(to_timestamp(ts))| + +-------------------+-----------------------------+ + |2015-07-22 10:00:00| 1437584400000000| + |2022-10-09 11:12:13| 1665339133000000| + +-------------------+-----------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") + """ + return _invoke_function_over_columns("unix_micros", col) + + +@_try_remote_functions +def unix_millis(col: "ColumnOrName") -> Column: + """Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. + Truncates higher levels of precision. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input column of values to convert. + A column that evaluates to a timestamp. + + Returns + ------- + :class:`~pyspark.sql.Column` + the number of milliseconds since 1970-01-01 00:00:00 UTC. + Returns a column that evaluates to a long. + + See Also + -------- + :meth:`pyspark.sql.functions.unix_date` + :meth:`pyspark.sql.functions.unix_seconds` + :meth:`pyspark.sql.functions.unix_micros` + :meth:`pyspark.sql.functions.unix_nanos` + :meth:`pyspark.sql.functions.timestamp_millis` + + Examples -------- >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") @@ -12202,22 +12791,345 @@ def try_to_timestamp(col: "ColumnOrName", format: Optional["ColumnOrName"] = Non @_try_remote_functions -def trunc(date: "ColumnOrName", format: str) -> Column: +def xpath(xml: "ColumnOrName", path: "ColumnOrName") -> Column: """ - Returns date truncated to the unit specified by the format. - - .. versionadded:: 1.5.0 + Returns a string array of values within the nodes of xml that match the XPath expression. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 - Parameters - ---------- - date : :class:`~pyspark.sql.Column` or column name - input column of values to truncate. - A column that evaluates to a date. - format : literal string - 'year', 'yyyy', 'yy' to truncate by year, + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [('b1b2b3c1c2',)], ['x']) + >>> df.select(sf.xpath(df.x, sf.lit('a/b/text()'))).show() + +--------------------+ + |xpath(x, a/b/text())| + +--------------------+ + | [b1, b2, b3]| + +--------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath", xml, path) + + +@_try_remote_functions +def xpath_boolean(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns true if the XPath expression evaluates to true, or if a matching node is found. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('1',)], ['x']) + >>> df.select(sf.xpath_boolean(df.x, sf.lit('a/b'))).show() + +---------------------+ + |xpath_boolean(x, a/b)| + +---------------------+ + | true| + +---------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_boolean", xml, path) + + +@_try_remote_functions +def xpath_double(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a double value, the value zero if no match is found, + or NaN if a match is found but the value is non-numeric. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_double(df.x, sf.lit('sum(a/b)'))).show() + +-------------------------+ + |xpath_double(x, sum(a/b))| + +-------------------------+ + | 3.0| + +-------------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_double", xml, path) + + +@_try_remote_functions +def xpath_number(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a double value, the value zero if no match is found, + or NaN if a match is found but the value is non-numeric. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [('12',)], ['x'] + ... ).select(sf.xpath_number('x', sf.lit('sum(a/b)'))).show() + +-------------------------+ + |xpath_number(x, sum(a/b))| + +-------------------------+ + | 3.0| + +-------------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_number", xml, path) + + +@_try_remote_functions +def xpath_float(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a float value, the value zero if no match is found, + or NaN if a match is found but the value is non-numeric. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_float(df.x, sf.lit('sum(a/b)'))).show() + +------------------------+ + |xpath_float(x, sum(a/b))| + +------------------------+ + | 3.0| + +------------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_float", xml, path) + + +@_try_remote_functions +def xpath_int(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns an integer value, or the value zero if no match is found, + or a match is found but the value is non-numeric. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_int(df.x, sf.lit('sum(a/b)'))).show() + +----------------------+ + |xpath_int(x, sum(a/b))| + +----------------------+ + | 3| + +----------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_int", xml, path) + + +@_try_remote_functions +def xpath_long(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a long integer value, or the value zero if no match is found, + or a match is found but the value is non-numeric. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_long(df.x, sf.lit('sum(a/b)'))).show() + +-----------------------+ + |xpath_long(x, sum(a/b))| + +-----------------------+ + | 3| + +-----------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_long", xml, path) + + +@_try_remote_functions +def xpath_short(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns a short integer value, or the value zero if no match is found, + or a match is found but the value is non-numeric. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('12',)], ['x']) + >>> df.select(sf.xpath_short(df.x, sf.lit('sum(a/b)'))).show() + +------------------------+ + |xpath_short(x, sum(a/b))| + +------------------------+ + | 3| + +------------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_string` + """ + return _invoke_function_over_columns("xpath_short", xml, path) + + +@_try_remote_functions +def xpath_string(xml: "ColumnOrName", path: "ColumnOrName") -> Column: + """ + Returns the text contents of the first xml node that matches the XPath expression. + + .. versionadded:: 3.5.0 + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('bcc',)], ['x']) + >>> df.select(sf.xpath_string(df.x, sf.lit('a/c'))).show() + +--------------------+ + |xpath_string(x, a/c)| + +--------------------+ + | cc| + +--------------------+ + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + """ + return _invoke_function_over_columns("xpath_string", xml, path) + + +@_try_remote_functions +def trunc(date: "ColumnOrName", format: str) -> Column: + """ + Returns date truncated to the unit specified by the format. + + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + date : :class:`~pyspark.sql.Column` or column name + input column of values to truncate. + A column that evaluates to a date. + format : literal string + 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month Other options are: 'week', 'quarter'. A column that evaluates to a string. @@ -13642,1838 +14554,1263 @@ def to_timestamp_ntz( return _invoke_function_over_columns("to_timestamp_ntz", timestamp) -@_try_remote_functions -def convert_timezone( - sourceTz: Optional[Column], targetTz: Column, sourceTs: "ColumnOrName" -) -> Column: - """ - Converts the timestamp without time zone `sourceTs` - from the `sourceTz` time zone to `targetTz`. +# ---------------------------- misc functions ---------------------------------- - .. versionadded:: 3.5.0 - Parameters - ---------- - sourceTz : :class:`~pyspark.sql.Column`, optional - The time zone for the input timestamp. If it is missed, - the current session time zone is used as the source time zone. - A column that evaluates to a string. - targetTz : :class:`~pyspark.sql.Column` - The time zone to which the input timestamp should be converted. - A column that evaluates to a string. - sourceTs : :class:`~pyspark.sql.Column` or column name - A timestamp without time zone. - A column that evaluates to a timestamp. +@_try_remote_functions +def current_catalog() -> Column: + """Returns the current catalog. - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains a timestamp for converted time zone. - Returns a column that evaluates to a timestamp. + .. versionadded:: 3.5.0 See Also -------- - :meth:`pyspark.sql.functions.current_timezone` + :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_schema` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.current_catalog()).show() + +-----------------+ + |current_catalog()| + +-----------------+ + | spark_catalog| + +-----------------+ + """ + return _invoke_function("current_catalog") - Example 1: Converts the timestamp without time zone `sourceTs`. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08 00:00:00',)], ['ts']) - >>> df.select( - ... '*', - ... sf.convert_timezone(None, sf.lit('Asia/Hong_Kong'), 'ts') - ... ).show() # doctest: +SKIP - +-------------------+--------------------------------------------------------+ - | ts|convert_timezone(current_timezone(), Asia/Hong_Kong, ts)| - +-------------------+--------------------------------------------------------+ - |2015-04-08 00:00:00| 2015-04-08 15:00:00| - +-------------------+--------------------------------------------------------+ +@_try_remote_functions +def current_path() -> Column: + """Returns the current SQL path as a comma-separated list of qualified schema names. - Example 2: Converts the timestamp with time zone `sourceTs`. + .. versionadded:: 4.2.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('2015-04-08 15:00:00',)], ['ts']) - >>> df.select( - ... '*', - ... sf.convert_timezone(sf.lit('Asia/Hong_Kong'), sf.lit('America/Los_Angeles'), df.ts) - ... ).show() - +-------------------+---------------------------------------------------------+ - | ts|convert_timezone(Asia/Hong_Kong, America/Los_Angeles, ts)| - +-------------------+---------------------------------------------------------+ - |2015-04-08 15:00:00| 2015-04-08 00:00:00| - +-------------------+---------------------------------------------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.current_catalog` + :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.current_schema` - >>> spark.conf.unset("spark.sql.session.timeZone") + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.current_path()).show() # doctest: +SKIP + +----------------------------------------------------+ + | current_path()| + +----------------------------------------------------+ + |system.builtin,system.session,spark_catalog.default | + +----------------------------------------------------+ """ - if sourceTz is None: - return _invoke_function_over_columns("convert_timezone", targetTz, sourceTs) - else: - return _invoke_function_over_columns("convert_timezone", sourceTz, targetTz, sourceTs) + return _invoke_function("current_path") @_try_remote_functions -def make_dt_interval( - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, -) -> Column: - """ - Make DayTimeIntervalType duration from days, hours, mins and secs. +def current_database() -> Column: + """Returns the current database. .. versionadded:: 3.5.0 - Parameters - ---------- - days : :class:`~pyspark.sql.Column` or column name, optional - The number of days, positive or negative. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The number of hours, positive or negative. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The number of minutes, positive or negative. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The number of seconds with the fractional part in microsecond precision. - A column that evaluates to a decimal. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains a DayTimeIntervalType duration. - Returns a column that evaluates to an interval. - See Also -------- - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.make_ym_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.current_catalog` + :meth:`pyspark.sql.functions.current_schema` Examples -------- - Example 1: Make DayTimeIntervalType duration from days, hours, mins and secs. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval(df.day, df.hour, df.min, df.sec)).show(truncate=False) - +---+----+---+--------+------------------------------------------+ - |day|hour|min|sec |make_dt_interval(day, hour, min, sec) | - +---+----+---+--------+------------------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 12:30:01.001001' DAY TO SECOND| - +---+----+---+--------+------------------------------------------+ - - Example 2: Make DayTimeIntervalType duration from days, hours and mins. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval(df.day, 'hour', df.min)).show(truncate=False) - +---+----+---+--------+-----------------------------------+ - |day|hour|min|sec |make_dt_interval(day, hour, min, 0)| - +---+----+---+--------+-----------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 12:30:00' DAY TO SECOND| - +---+----+---+--------+-----------------------------------+ - - Example 3: Make DayTimeIntervalType duration from days and hours. + >>> spark.range(1).select(sf.current_database()).show() + +----------------+ + |current_schema()| + +----------------+ + | default| + +----------------+ + """ + return _invoke_function("current_database") - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval(df.day, df.hour)).show(truncate=False) - +---+----+---+--------+-----------------------------------+ - |day|hour|min|sec |make_dt_interval(day, hour, 0, 0) | - +---+----+---+--------+-----------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 12:00:00' DAY TO SECOND| - +---+----+---+--------+-----------------------------------+ - Example 4: Make DayTimeIntervalType duration from days. +@_try_remote_functions +def current_schema() -> Column: + """Returns the current database. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) - >>> df.select('*', sf.make_dt_interval('day')).show(truncate=False) - +---+----+---+--------+-----------------------------------+ - |day|hour|min|sec |make_dt_interval(day, 0, 0, 0) | - +---+----+---+--------+-----------------------------------+ - |1 |12 |30 |1.001001|INTERVAL '1 00:00:00' DAY TO SECOND| - +---+----+---+--------+-----------------------------------+ + .. versionadded:: 3.5.0 - Example 5: Make empty interval. + See Also + -------- + :meth:`pyspark.sql.functions.current_catalog` + :meth:`pyspark.sql.functions.current_database` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_dt_interval()).show(truncate=False) - +-----------------------------------+ - |make_dt_interval(0, 0, 0, 0) | - +-----------------------------------+ - |INTERVAL '0 00:00:00' DAY TO SECOND| - +-----------------------------------+ + >>> spark.range(1).select(sf.current_schema()).show() + +----------------+ + |current_schema()| + +----------------+ + | default| + +----------------+ """ - _days = lit(0) if days is None else days - _hours = lit(0) if hours is None else hours - _mins = lit(0) if mins is None else mins - _secs = lit(decimal.Decimal(0)) if secs is None else secs - return _invoke_function_over_columns("make_dt_interval", _days, _hours, _mins, _secs) + return _invoke_function("current_schema") @_try_remote_functions -def try_make_interval( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - weeks: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, -) -> Column: - """ - This is a special version of `make_interval` that performs the same operation, but returns a - NULL value instead of raising an error if interval cannot be created. - - .. versionadded:: 4.0.0 - - Parameters - ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative. - A column that evaluates to an integer. - weeks : :class:`~pyspark.sql.Column` or column name, optional - The number of weeks, positive or negative. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The number of days, positive or negative. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The number of hours, positive or negative. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The number of minutes, positive or negative. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The number of seconds with the fractional part in microsecond precision. - A column that evaluates to a decimal. +def current_user() -> Column: + """Returns the current database. - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains an interval. - Returns a column that evaluates to an interval. + .. versionadded:: 3.5.0 See Also -------- - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.make_dt_interval` - :meth:`pyspark.sql.functions.make_ym_interval` + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.session_user` Examples -------- - Example 1: Try make interval from years, months, weeks, days, hours, mins and secs. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_interval(df.year, df.month, 'week', df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +---------------------------------------------------------------+ - |try_make_interval(year, month, week, day, hour, min, sec) | - +---------------------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| - +---------------------------------------------------------------+ + >>> spark.range(1).select(sf.current_user()).show() # doctest: +SKIP + +--------------+ + |current_user()| + +--------------+ + | ruifeng.zheng| + +--------------+ + """ + return _invoke_function("current_user") - Example 2: Try make interval from years, months, weeks, days, hours and mins. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) - ... ).show(truncate=False) - +-------------------------------------------------------+ - |try_make_interval(year, month, week, day, hour, min, 0)| - +-------------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes | - +-------------------------------------------------------+ +@_try_remote_functions +def user() -> Column: + """Returns the current database. - Example 3: Try make interval from years, months, weeks, days and hours. + .. versionadded:: 3.5.0 + + See Also + -------- + :meth:`pyspark.sql.functions.current_user` + :meth:`pyspark.sql.functions.session_user` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour) - ... ).show(truncate=False) - +-----------------------------------------------------+ - |try_make_interval(year, month, week, day, hour, 0, 0)| - +-----------------------------------------------------+ - |100 years 11 months 8 days 12 hours | - +-----------------------------------------------------+ + >>> spark.range(1).select(sf.user()).show() # doctest: +SKIP + +--------------+ + | user()| + +--------------+ + | ruifeng.zheng| + +--------------+ + """ + return _invoke_function("user") - Example 4: Try make interval from years, months, weeks and days. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year, 'month', df.week, df.day)).show(truncate=False) - +--------------------------------------------------+ - |try_make_interval(year, month, week, day, 0, 0, 0)| - +--------------------------------------------------+ - |100 years 11 months 8 days | - +--------------------------------------------------+ +@_try_remote_functions +def session_user() -> Column: + """Returns the user name of current execution context. - Example 5: Try make interval from years, months and weeks. + .. versionadded:: 4.0.0 + + See Also + -------- + :meth:`pyspark.sql.functions.user` + :meth:`pyspark.sql.functions.current_user` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year, 'month', df.week)).show(truncate=False) - +------------------------------------------------+ - |try_make_interval(year, month, week, 0, 0, 0, 0)| - +------------------------------------------------+ - |100 years 11 months 7 days | - +------------------------------------------------+ + >>> spark.range(1).select(sf.session_user()).show() # doctest: +SKIP + +--------------+ + |session_user()| + +--------------+ + | ruifeng.zheng| + +--------------+ + """ + return _invoke_function("session_user") - Example 6: Try make interval from years and months. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year, 'month')).show(truncate=False) - +---------------------------------------------+ - |try_make_interval(year, month, 0, 0, 0, 0, 0)| - +---------------------------------------------+ - |100 years 11 months | - +---------------------------------------------+ +@_try_remote_functions +def uuid(seed: Optional[Union[Column, int]] = None) -> Column: + """Returns an universally unique identifier (UUID) string. + The value is returned as a canonical UUID 36-character string. - Example 7: Try make interval from years. + .. versionadded:: 4.1.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.try_make_interval(df.year)).show(truncate=False) - +-----------------------------------------+ - |try_make_interval(year, 0, 0, 0, 0, 0, 0)| - +-----------------------------------------+ - |100 years | - +-----------------------------------------+ + Parameters + ---------- + seed : :class:`~pyspark.sql.Column` or int + Optional random number seed to use. - Example 8: Try make empty interval. + Examples + -------- + Example 1: Generate UUIDs with random seed - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.try_make_interval()).show(truncate=False) - +--------------------------------------+ - |try_make_interval(0, 0, 0, 0, 0, 0, 0)| - +--------------------------------------+ - |0 seconds | - +--------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> spark.range(5).select(sf.uuid()).show(truncate=False) # doctest: +SKIP + +------------------------------------+ + |uuid() | + +------------------------------------+ + |627ae05e-b319-42b5-b4e4-71c8c9754dd1| + |f781cce5-a2e2-464d-bc8b-426ff448e404| + |15e2e66e-8416-4ea2-af3c-409363408189| + |fb1d6178-7676-4791-baa9-f2ddcc494515| + |d48665e8-2657-4c6b-b7c8-8ae0cd646e41| + +------------------------------------+ - Example 9: Try make interval from years with overflow. + Example 2: Generate UUIDs with a specified seed - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.try_make_interval(sf.lit(2147483647))).show(truncate=False) - +-----------------------------------------------+ - |try_make_interval(2147483647, 0, 0, 0, 0, 0, 0)| - +-----------------------------------------------+ - |NULL | - +-----------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> spark.range(0, 5, 1, 1).select(sf.uuid(seed=123)).show(truncate=False) + +------------------------------------+ + |uuid() | + +------------------------------------+ + |4c99192d-23d6-4d88-b814-a634398120f0| + |af506873-3c53-41e3-8354-a24856b8de8a| + |7b4b370e-e867-47e2-93c0-f6990463a12d| + |1c4d1733-ff1a-4a6c-b144-0b0345adf0d0| + |7478f235-f8bc-4112-8e59-a28f50e46890| + +------------------------------------+ """ - _years = lit(0) if years is None else years - _months = lit(0) if months is None else months - _weeks = lit(0) if weeks is None else weeks - _days = lit(0) if days is None else days - _hours = lit(0) if hours is None else hours - _mins = lit(0) if mins is None else mins - _secs = lit(decimal.Decimal(0)) if secs is None else secs - return _invoke_function_over_columns( - "try_make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs - ) + from pyspark.sql.classic.column import _to_java_column + + if seed is None: + return _invoke_function("uuid") + else: + return _invoke_function("uuid", _to_java_column(lit(seed))) @_try_remote_functions -def make_interval( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - weeks: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, -) -> Column: +def crc32(col: "ColumnOrName") -> Column: """ - Make interval from years, months, weeks, days, hours, mins and secs. + Calculates the cyclic redundancy check value (CRC32) of a binary column and + returns the value as a bigint. - .. versionadded:: 3.5.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative. - A column that evaluates to an integer. - weeks : :class:`~pyspark.sql.Column` or column name, optional - The number of weeks, positive or negative. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The number of days, positive or negative. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The number of hours, positive or negative. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The number of minutes, positive or negative. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The number of seconds with the fractional part in microsecond precision. - A column that evaluates to a decimal. + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an interval. - Returns a column that evaluates to an interval. + the column for computed results. + Returns a column that evaluates to a long. - See Also - -------- - :meth:`pyspark.sql.functions.make_dt_interval` - :meth:`pyspark.sql.functions.make_ym_interval` - :meth:`pyspark.sql.functions.try_make_interval` + .. versionadded:: 1.5.0 Examples -------- - Example 1: Make interval from years, months, weeks, days, hours, mins and secs. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +---------------------------------------------------------------+ - |make_interval(year, month, week, day, hour, min, sec) | - +---------------------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| - +---------------------------------------------------------------+ + >>> df = spark.createDataFrame([('ABC',)], ['a']) + >>> df.select('*', sf.crc32('a')).show(truncate=False) + +---+----------+ + |a |crc32(a) | + +---+----------+ + |ABC|2743272264| + +---+----------+ + """ + return _invoke_function_over_columns("crc32", col) - Example 2: Make interval from years, months, weeks, days, hours and mins. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) - ... ).show(truncate=False) - +---------------------------------------------------+ - |make_interval(year, month, week, day, hour, min, 0)| - +---------------------------------------------------+ - |100 years 11 months 8 days 12 hours 30 minutes | - +---------------------------------------------------+ +@_try_remote_functions +def md5(col: "ColumnOrName") -> Column: + """Calculates the MD5 digest and returns the value as a 32 character hex string. - Example 3: Make interval from years, months, weeks, days and hours. + .. versionadded:: 1.5.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour) - ... ).show(truncate=False) - +-------------------------------------------------+ - |make_interval(year, month, week, day, hour, 0, 0)| - +-------------------------------------------------+ - |100 years 11 months 8 days 12 hours | - +-------------------------------------------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. - Example 4: Make interval from years, months, weeks and days. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a binary. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Returns a column that evaluates to a string. + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year, df.month, 'week', df.day)).show(truncate=False) - +----------------------------------------------+ - |make_interval(year, month, week, day, 0, 0, 0)| - +----------------------------------------------+ - |100 years 11 months 8 days | - +----------------------------------------------+ + >>> df = spark.createDataFrame([('ABC',)], ['a']) + >>> df.select('*', sf.md5('a')).show(truncate=False) + +---+--------------------------------+ + |a |md5(a) | + +---+--------------------------------+ + |ABC|902fbdd2b1df0c4f70b4a5d23525e932| + +---+--------------------------------+ + """ + return _invoke_function_over_columns("md5", col) - Example 5: Make interval from years, months and weeks. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year, df.month, 'week')).show(truncate=False) - +--------------------------------------------+ - |make_interval(year, month, week, 0, 0, 0, 0)| - +--------------------------------------------+ - |100 years 11 months 7 days | - +--------------------------------------------+ +@_try_remote_functions +def xxh3_64(col: "ColumnOrName") -> Column: + """Returns a 64-bit hash value of the argument using the XXH3 algorithm. - Example 6: Make interval from years and months. + Unlike :func:`xxhash64`, which hashes one or more columns structurally, this hashes the raw + bytes of a single value with seed 0, so its result is byte compatible with the reference XXH3. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year, df.month)).show(truncate=False) - +-----------------------------------------+ - |make_interval(year, month, 0, 0, 0, 0, 0)| - +-----------------------------------------+ - |100 years 11 months | - +-----------------------------------------+ + .. versionadded:: 4.4.0 - Example 7: Make interval from years. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The target column to hash, which must have string or binary type. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], - ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) - >>> df.select(sf.make_interval(df.year)).show(truncate=False) - +-------------------------------------+ - |make_interval(year, 0, 0, 0, 0, 0, 0)| - +-------------------------------------+ - |100 years | - +-------------------------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + Returns a column that evaluates to a long. - Example 8: Make empty interval. + See Also + -------- + :meth:`pyspark.sql.functions.xxh3_128` + :meth:`pyspark.sql.functions.xxhash64` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_interval()).show(truncate=False) - +----------------------------------+ - |make_interval(0, 0, 0, 0, 0, 0, 0)| - +----------------------------------+ - |0 seconds | - +----------------------------------+ + >>> df = spark.createDataFrame([('Spark',)], ['a']) + >>> df.select(sf.xxh3_64('a').alias('h')).collect() + [Row(h=80997306238743657)] """ - _years = lit(0) if years is None else years - _months = lit(0) if months is None else months - _weeks = lit(0) if weeks is None else weeks - _days = lit(0) if days is None else days - _hours = lit(0) if hours is None else hours - _mins = lit(0) if mins is None else mins - _secs = lit(decimal.Decimal(0)) if secs is None else secs - return _invoke_function_over_columns( - "make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs - ) + return _invoke_function_over_columns("xxh3_64", col) @_try_remote_functions -def make_time(hour: "ColumnOrName", minute: "ColumnOrName", second: "ColumnOrName") -> Column: - """ - Create time from hour, minute and second fields. For invalid inputs it will throw an error. +def xxh3_128(col: "ColumnOrName") -> Column: + """Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. - .. versionadded:: 4.1.0 + .. versionadded:: 4.4.0 Parameters ---------- - hour : :class:`~pyspark.sql.Column` or column name - The hour to represent, from 0 to 23. - A column that evaluates to an integer. - minute : :class:`~pyspark.sql.Column` or column name - The minute to represent, from 0 to 59. - A column that evaluates to an integer. - second : :class:`~pyspark.sql.Column` or column name - The second to represent, from 0 to 59.999999. - A column that evaluates to a decimal. + col : :class:`~pyspark.sql.Column` or column name + The target column to hash, which must have string or binary type. Returns ------- :class:`~pyspark.sql.Column` - A column representing the created time. - Returns a column that evaluates to a time. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.xxh3_64` + :meth:`pyspark.sql.functions.md5` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(6, 30, 45.887)], ["hour", "minute", "second"]) - >>> df.select(sf.make_time("hour", "minute", "second").alias("time")).show() - +------------+ - | time| - +------------+ - |06:30:45.887| - +------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark',)], ['a']) + >>> df.select(sf.xxh3_128('a').alias('h')).collect() + [Row(h='7d57dd84c60c86ca1f4e82ab91a12b5e')] """ - return _invoke_function_over_columns("make_time", hour, minute, second) + return _invoke_function_over_columns("xxh3_128", col) @_try_remote_functions -def time_from_seconds(col: "ColumnOrName") -> Column: - """ - Creates a TIME value from seconds since midnight (supports fractional seconds). +def sha1(col: "ColumnOrName") -> Column: + """Returns the hex string result of SHA-1. - .. versionadded:: 4.2.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Seconds since midnight (0 to 86399.999999). - A column that evaluates to a numeric. + target column to compute on. + A column that evaluates to a binary. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.sha` + :meth:`pyspark.sql.functions.sha2` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(52200.5,)], ['seconds']) - >>> df.select(sf.time_from_seconds('seconds')).show() - +--------------------------+ - |time_from_seconds(seconds)| - +--------------------------+ - | 14:30:00.5| - +--------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ABC',)], ['a']) + >>> df.select('*', sf.sha1('a')).show(truncate=False) + +---+----------------------------------------+ + |a |sha1(a) | + +---+----------------------------------------+ + |ABC|3c01bdbb26f358bab27f267924aa2c9a03fcfdb8| + +---+----------------------------------------+ """ - return _invoke_function_over_columns("time_from_seconds", col) + return _invoke_function_over_columns("sha1", col) @_try_remote_functions -def time_from_millis(col: "ColumnOrName") -> Column: - """ - Creates a TIME value from milliseconds since midnight. +def sha2(col: "ColumnOrName", numBits: int) -> Column: + """Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, + and SHA-512). The numBits indicates the desired bit length of the result, which must have a + value of 224, 256, 384, 512, or 0 (which is equivalent to 256). - .. versionadded:: 4.2.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - Milliseconds since midnight (0 to 86399999). - A column that evaluates to an integral. + target column to compute on. + A column that evaluates to a binary. + numBits : int + the desired bit length of the result, which must have a + value of 224, 256, 384, 512, or 0 (which is equivalent to 256). + A column that evaluates to an integer. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column for computed results. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.sha` + :meth:`pyspark.sql.functions.sha1` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(52200500,)], ['millis']) - >>> df.select(sf.time_from_millis('millis')).show() - +------------------------+ - |time_from_millis(millis)| - +------------------------+ - | 14:30:00.5| - +------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([['Alice'], ['Bob']], ['name']) + >>> df.select('*', sf.sha2('name', 256)).show(truncate=False) + +-----+----------------------------------------------------------------+ + |name |sha2(name, 256) | + +-----+----------------------------------------------------------------+ + |Alice|3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043| + |Bob |cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961| + +-----+----------------------------------------------------------------+ """ - return _invoke_function_over_columns("time_from_millis", col) + from pyspark.sql.classic.column import _to_java_column + + if numBits not in [0, 224, 256, 384, 512]: + raise PySparkValueError( + errorClass="VALUE_NOT_ALLOWED", + messageParameters={ + "arg_name": "numBits", + "allowed_values": "[0, 224, 256, 384, 512]", + }, + ) + return _invoke_function("sha2", _to_java_column(col), numBits) @_try_remote_functions -def time_from_micros(col: "ColumnOrName") -> Column: - """ - Creates a TIME value from microseconds since midnight. +def hash(*cols: "ColumnOrName") -> Column: + """Calculates the hash code of given columns, and returns the result as an int column. - .. versionadded:: 4.2.0 + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Microseconds since midnight (0 to 86399999999). - A column that evaluates to an integral. + cols : :class:`~pyspark.sql.Column` or column name + one or more columns to compute on. + Each a column of any type. + + Returns + ------- + :class:`~pyspark.sql.Column` + hash value as int column. + Returns a column that evaluates to an integer. + + See Also + -------- + :meth:`pyspark.sql.functions.xxhash64` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(52200500000,)], ['micros']) - >>> df.select(sf.time_from_micros('micros')).show() - +------------------------+ - |time_from_micros(micros)| - +------------------------+ - | 14:30:00.5| - +------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) + >>> df.select('*', sf.hash('c1')).show() + +---+---+----------+ + | c1| c2| hash(c1)| + +---+---+----------+ + |ABC|DEF|-757602832| + +---+---+----------+ + + >>> df.select('*', sf.hash('c1', df.c2)).show() + +---+---+------------+ + | c1| c2|hash(c1, c2)| + +---+---+------------+ + |ABC|DEF| 599895104| + +---+---+------------+ + + >>> df.select('*', sf.hash('*')).show() + +---+---+------------+ + | c1| c2|hash(c1, c2)| + +---+---+------------+ + |ABC|DEF| 599895104| + +---+---+------------+ """ - return _invoke_function_over_columns("time_from_micros", col) + return _invoke_function_over_seq_of_columns("hash", cols) @_try_remote_functions -def time_to_seconds(col: "ColumnOrName") -> Column: - """ - Extracts seconds from TIME value (returns DECIMAL to preserve fractional seconds). +def xxhash64(*cols: "ColumnOrName") -> Column: + """Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, + and returns the result as a long column. The hash computation uses an initial seed of 42. - .. versionadded:: 4.2.0 + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - TIME value to convert. + cols : :class:`~pyspark.sql.Column` or column name + one or more columns to compute on. + Each a column of any type. + + Returns + ------- + :class:`~pyspark.sql.Column` + hash value as long column. + Returns a column that evaluates to a long. + + See Also + -------- + :meth:`pyspark.sql.functions.hash` + :meth:`pyspark.sql.functions.xxh3_64` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") - >>> df.select(sf.time_to_seconds('time')).show() - +---------------------+ - |time_to_seconds(time)| - +---------------------+ - | 52200.500000| - +---------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) + >>> df.select('*', sf.xxhash64('c1')).show() + +---+---+-------------------+ + | c1| c2| xxhash64(c1)| + +---+---+-------------------+ + |ABC|DEF|4105715581806190027| + +---+---+-------------------+ + + >>> df.select('*', sf.xxhash64('c1', df.c2)).show() + +---+---+-------------------+ + | c1| c2| xxhash64(c1, c2)| + +---+---+-------------------+ + |ABC|DEF|3233247871021311208| + +---+---+-------------------+ + + >>> df.select('*', sf.xxhash64('*')).show() + +---+---+-------------------+ + | c1| c2| xxhash64(c1, c2)| + +---+---+-------------------+ + |ABC|DEF|3233247871021311208| + +---+---+-------------------+ """ - return _invoke_function_over_columns("time_to_seconds", col) + return _invoke_function_over_seq_of_columns("xxhash64", cols) @_try_remote_functions -def time_to_millis(col: "ColumnOrName") -> Column: +def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None) -> Column: """ - Extracts milliseconds from TIME value. + Returns `null` if the input column is `true`; throws an exception + with the provided error message otherwise. - .. versionadded:: 4.2.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - TIME value to convert. + column name or column that represents the input column to test. + A column that evaluates to a boolean. + errMsg : :class:`~pyspark.sql.Column` or literal string, optional + A Python string literal or column containing the error message. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + `null` if the input column is `true` otherwise throws an error with specified message. + Returns a column that always evaluates to NULL. + + See Also + -------- + :meth:`pyspark.sql.functions.raise_error` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") - >>> df.select(sf.time_to_millis('time')).show() - +--------------------+ - |time_to_millis(time)| - +--------------------+ - | 52200500| - +--------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(0, 1)], ['a', 'b']) + >>> df.select('*', sf.assert_true(df.a < df.b)).show() + +---+---+--------------------------------------------+ + | a| b|assert_true((a < b), '(a < b)' is not true!)| + +---+---+--------------------------------------------+ + | 0| 1| NULL| + +---+---+--------------------------------------------+ + + >>> df.select('*', sf.assert_true(df.a < df.b, df.a)).show() + +---+---+-----------------------+ + | a| b|assert_true((a < b), a)| + +---+---+-----------------------+ + | 0| 1| NULL| + +---+---+-----------------------+ + + >>> df.select('*', sf.assert_true(df.a < df.b, 'error')).show() + +---+---+---------------------------+ + | a| b|assert_true((a < b), error)| + +---+---+---------------------------+ + | 0| 1| NULL| + +---+---+---------------------------+ + + >>> df.select('*', sf.assert_true(df.a > df.b, 'My error msg')).show() # doctest: +SKIP + ... + java.lang.RuntimeException: My error msg + ... """ - return _invoke_function_over_columns("time_to_millis", col) + errMsg = _enum_to_value(errMsg) + if errMsg is None: + return _invoke_function_over_columns("assert_true", col) + if not isinstance(errMsg, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "errMsg", + "arg_type": type(errMsg).__name__, + }, + ) + return _invoke_function_over_columns("assert_true", col, lit(errMsg)) @_try_remote_functions -def time_to_micros(col: "ColumnOrName") -> Column: +def raise_error(errMsg: Union[Column, str]) -> Column: """ - Extracts microseconds from TIME value. + Throws an exception with the provided error message. - .. versionadded:: 4.2.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - TIME value to convert. + errMsg : :class:`~pyspark.sql.Column` or literal string + A Python string literal or column containing the error message. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + throws an error with specified message. + Returns a column that always evaluates to NULL. + + See Also + -------- + :meth:`pyspark.sql.functions.assert_true` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") - >>> df.select(sf.time_to_micros('time')).show() - +--------------------+ - |time_to_micros(time)| - +--------------------+ - | 52200500000| - +--------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.raise_error("My error message")).show() # doctest: +SKIP + ... + java.lang.RuntimeException: My error message + ... """ - return _invoke_function_over_columns("time_to_micros", col) - - -@overload -def make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... - - -@overload -def make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: "ColumnOrName", -) -> Column: ... - - -@overload -def make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... + errMsg = _enum_to_value(errMsg) + if not isinstance(errMsg, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "errMsg", + "arg_type": type(errMsg).__name__, + }, + ) + return _invoke_function_over_columns("raise_error", lit(errMsg)) -@overload -def make_timestamp( - *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" -) -> Column: ... +# ---------------------- String/Binary functions ------------------------------ @_try_remote_functions -def make_timestamp( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - timezone: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, -) -> Column: +def upper(col: "ColumnOrName") -> Column: """ - Create timestamp from years, months, days, hours, mins, secs, and (optional) timezone fields. - Alternatively, create timestamp from date, time, and (optional) timezone fields. - The result data type is consistent with the value of configuration `spark.sql.timestampType`. - If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL - on invalid inputs. Otherwise, it will throw an error instead. + Converts a string expression to upper case. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or column name, optional - The time zone identifier. For example, CET, UTC, and etc. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. A column that evaluates to a string. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a timestamp. - Returns a column that evaluates to a timestamp. + upper case values. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.lower` + :meth:`pyspark.sql.functions.ucase` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - - Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.upper("value")).show() + +----------+------------+ + | value|upper(value)| + +----------+------------+ + | Spark| SPARK| + | PySpark| PYSPARK| + |Pandas API| PANDAS API| + +----------+------------+ + """ + return _invoke_function_over_columns("upper", col) - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') - ... ).show(truncate=False) - +----------------------------------------------------+ - |make_timestamp(year, month, day, hour, min, sec, tz)| - +----------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +----------------------------------------------------+ - Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). +@_try_remote_functions +def lower(col: "ColumnOrName") -> Column: + """ + Converts a string expression to lower case. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +------------------------------------------------+ - |make_timestamp(year, month, day, hour, min, sec)| - +------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +------------------------------------------------+ + .. versionadded:: 1.5.0 - Example 3: Make timestamp from date, time, and timezone. + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time"), - ... sf.lit("CET").alias("tz") - ... ) - >>> df.select( - ... sf.make_timestamp(date=df.date, time=df.time, timezone=df.tz) - ... ).show(truncate=False) - +------------------------------+ - |make_timestamp(date, time, tz)| - +------------------------------+ - |2014-12-27 21:30:45.887 | - +------------------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. - Example 4: Make timestamp from date and time (without timezone). + Returns + ------- + :class:`~pyspark.sql.Column` + lower case values. + Returns a column that evaluates to a string. - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time") - ... ) - >>> df.select(sf.make_timestamp(date=df.date, time=df.time)).show(truncate=False) - +--------------------------+ - |make_timestamp(date, time)| - +--------------------------+ - |2014-12-28 06:30:45.887 | - +--------------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.upper` + :meth:`pyspark.sql.functions.lcase` - >>> spark.conf.unset("spark.sql.session.timeZone") + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.lower("value")).show() + +----------+------------+ + | value|lower(value)| + +----------+------------+ + | Spark| spark| + | PySpark| pyspark| + |Pandas API| pandas api| + +----------+------------+ """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) - else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "make_timestamp", - _ensure_column_or_name(date), - _ensure_column_or_name(time), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) - + return _invoke_function_over_columns("lower", col) -@overload -def try_make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... +@_try_remote_functions +def ascii(col: "ColumnOrName") -> Column: + """ + Computes the numeric value of the first character of the string column. -@overload -def try_make_timestamp( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: "ColumnOrName", -) -> Column: ... + .. versionadded:: 1.5.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. -@overload -def try_make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + Returns + ------- + :class:`~pyspark.sql.Column` + numeric value. + Returns a column that evaluates to an integer. -@overload -def try_make_timestamp( - *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" -) -> Column: ... + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.ascii("value")).show() + +----------+------------+ + | value|ascii(value)| + +----------+------------+ + | Spark| 83| + | PySpark| 80| + |Pandas API| 80| + +----------+------------+ + """ + return _invoke_function_over_columns("ascii", col) @_try_remote_functions -def try_make_timestamp( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - timezone: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, -) -> Column: +def base64(col: "ColumnOrName") -> Column: """ - Try to create timestamp from years, months, days, hours, mins, secs and (optional) timezone - fields. Alternatively, try to create timestamp from date, time, and (optional) timezone fields. - The result data type is consistent with the value of configuration `spark.sql.timestampType`. - The function returns NULL on invalid inputs. + Computes the BASE64 encoding of a binary column and returns it as a string column. - .. versionadded:: 4.0.0 + .. versionadded:: 1.5.0 - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or column name, optional - The time zone identifier. For example, CET, UTC, and etc. - A column that evaluates to a string. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a timestamp or NULL in case of an error. - Returns a column that evaluates to a timestamp. + BASE64 encoding of string value. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.unbase64` + :meth:`pyspark.sql.functions.to_base32` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Spark", "PySpark", "Pandas API"], "STRING") + >>> df.select("*", sf.base64("value")).show() + +----------+----------------+ + | value| base64(value)| + +----------+----------------+ + | Spark| U3Bhcms=| + | PySpark| UHlTcGFyaw==| + |Pandas API|UGFuZGFzIEFQSQ==| + +----------+----------------+ + """ + return _invoke_function_over_columns("base64", col) - Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') - ... ).show(truncate=False) - +----------------------------------------------------+ - |try_make_timestamp(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +----------------------------------------------------+ +@_try_remote_functions +def to_base32(col: "ColumnOrName") -> Column: + """ + Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a + string column. - Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). + .. versionadded:: 4.3.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |try_make_timestamp(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +----------------------------------------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a binary. - Example 3: Make timestamp with invalid input. + Returns + ------- + :class:`~pyspark.sql.Column` + BASE32 encoding of the binary value. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.from_base32` + :meth:`pyspark.sql.functions.base64` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |try_make_timestamp(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |NULL | - +----------------------------------------------------+ + >>> df = spark.createDataFrame([(b"foobar",)], ["value"]) + >>> df.select(sf.to_base32("value").alias("r")).collect() + [Row(r='MZXW6YTBOI======')] + """ + return _invoke_function_over_columns("to_base32", col) - Example 4: Make timestamp from date, time, and timezone. - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time"), - ... sf.lit("CET").alias("tz") - ... ) - >>> df.select( - ... sf.try_make_timestamp(date=df.date, time=df.time, timezone=df.tz) - ... ).show(truncate=False) - +----------------------------------+ - |try_make_timestamp(date, time, tz)| - +----------------------------------+ - |2014-12-27 21:30:45.887 | - +----------------------------------+ +@_try_remote_functions +def unbase64(col: "ColumnOrName") -> Column: + """ + Decodes a BASE64 encoded string column and returns it as a binary column. - Example 5: Make timestamp from date and time (without timezone). + .. versionadded:: 1.5.0 - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time") - ... ) - >>> df.select(sf.try_make_timestamp(date=df.date, time=df.time)).show(truncate=False) - +------------------------------+ - |try_make_timestamp(date, time)| - +------------------------------+ - |2014-12-28 06:30:45.887 | - +------------------------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> spark.conf.unset("spark.sql.session.timeZone") + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + decoded binary value. + Returns a column that evaluates to a binary. + + See Also + -------- + :meth:`pyspark.sql.functions.base64` + :meth:`pyspark.sql.functions.from_base32` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["U3Bhcms=", "UHlTcGFyaw==", "UGFuZGFzIEFQSQ=="], "STRING") + >>> df.select("*", sf.unbase64("value")).show(truncate=False) + +----------------+-------------------------------+ + |value |unbase64(value) | + +----------------+-------------------------------+ + |U3Bhcms= |[53 70 61 72 6B] | + |UHlTcGFyaw== |[50 79 53 70 61 72 6B] | + |UGFuZGFzIEFQSQ==|[50 61 6E 64 61 73 20 41 50 49]| + +----------------+-------------------------------+ """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "try_make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "try_make_timestamp", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) - else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - if timezone is not None: - return _invoke_function_over_columns( - "try_make_timestamp", - _ensure_column_or_name(date), - _ensure_column_or_name(time), - _ensure_column_or_name(timezone), - ) - else: - return _invoke_function_over_columns( - "try_make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) + return _invoke_function_over_columns("unbase64", col) @_try_remote_functions -def make_timestamp_ltz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: Optional["ColumnOrName"] = None, -) -> Column: +def from_base32(col: "ColumnOrName") -> Column: """ - Create the current timestamp with local time zone from years, months, days, hours, mins, - secs and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, - the function returns NULL on invalid inputs. Otherwise, it will throw an error instead. + Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary + column. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - years : :class:`~pyspark.sql.Column` or str - The year to represent, from 1 to 9999. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or str - The month-of-year to represent, from 1 (January) to 12 (December). - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or str - The day-of-month to represent, from 1 to 31. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or str - The hour-of-day to represent, from 0 to 23. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or str - The minute-of-hour to represent, from 0 to 59. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or str - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13 , or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or str, optional - The time zone identifier. For example, CET, UTC and etc. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a current timestamp. - Returns a column that evaluates to a timestamp. + decoded binary value. + Returns a column that evaluates to a binary. See Also -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.to_base32` + :meth:`pyspark.sql.functions.unbase64` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - - Example 1: Make the current timestamp from years, months, days, hours, mins and secs. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec, 'tz') - ... ).show(truncate=False) - +--------------------------------------------------------+ - |make_timestamp_ltz(year, month, day, hour, min, sec, tz)| - +--------------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +--------------------------------------------------------+ - - Example 2: Make the current timestamp without timezone. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |make_timestamp_ltz(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +----------------------------------------------------+ - - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> df = spark.createDataFrame([("MZXW6YTBOI======",)], ["value"]) + >>> df.select(sf.from_base32("value").alias("r")).collect() + [Row(r=b'foobar')] """ - if timezone is not None: - return _invoke_function_over_columns( - "make_timestamp_ltz", years, months, days, hours, mins, secs, timezone - ) - else: - return _invoke_function_over_columns( - "make_timestamp_ltz", years, months, days, hours, mins, secs - ) + return _invoke_function_over_columns("from_base32", col) @_try_remote_functions -def try_make_timestamp_ltz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", - timezone: Optional["ColumnOrName"] = None, -) -> Column: +def ltrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: """ - Try to create the current timestamp with local time zone from years, months, days, hours, mins, - secs and timezone fields. - The function returns NULL on invalid inputs. + Trim the spaces from left end for the specified string value. - .. versionadded:: 4.0.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name - The year to represent, from 1 to 9999. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name - The month-of-year to represent, from 1 (January) to 12 (December). - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name - The day-of-month to represent, from 1 to 31. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name - The hour-of-day to represent, from 0 to 23. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name - The minute-of-hour to represent, from 0 to 59. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13 , or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - A column that evaluates to a decimal. - timezone : :class:`~pyspark.sql.Column` or column name, optional - The time zone identifier. For example, CET, UTC and etc. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or column name, optional + The trim string characters to trim, the default value is a single space. A column that evaluates to a string. + .. versionadded:: 4.0.0 + Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a current timestamp, or NULL in case of an error. - Returns a column that evaluates to a timestamp. + left trimmed values. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.trim` + :meth:`pyspark.sql.functions.rtrim` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - - Example 1: Make the current timestamp from years, months, days, hours, mins and secs. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec, 'tz') - ... ).show(truncate=False) - +------------------------------------------------------------+ - |try_make_timestamp_ltz(year, month, day, hour, min, sec, tz)| - +------------------------------------------------------------+ - |2014-12-27 21:30:45.887 | - +------------------------------------------------------------+ + Example 1: Trim the spaces - Example 2: Make the current timestamp without timezone. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select("*", sf.ltrim("value")).show() + +--------+------------+ + | value|ltrim(value)| + +--------+------------+ + | Spark| Spark| + | Spark | Spark | + | Spark| Spark| + +--------+------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ltz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +--------------------------------------------------------+ + Example 2: Trim specified characters - Example 3: Make the current timestamp with invalid input. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") + >>> df.select("*", sf.ltrim("value", sf.lit("*"))).show() + +--------+--------------------------+ + | value|TRIM(LEADING * FROM value)| + +--------+--------------------------+ + |***Spark| Spark| + | Spark**| Spark**| + | *Spark| Spark| + +--------+--------------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], - ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) - >>> df.select( - ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ltz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |NULL | - +--------------------------------------------------------+ + Example 3: Trim a column containing different characters - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) + >>> df.select("*", sf.ltrim("value", "t")).show() + +--------+---+--------------------------+ + | value| t|TRIM(LEADING t FROM value)| + +--------+---+--------------------------+ + |**Spark*| *| Spark*| + |==Spark=| =| Spark=| + +--------+---+--------------------------+ """ - if timezone is not None: - return _invoke_function_over_columns( - "try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone - ) + if trim is not None: + return _invoke_function_over_columns("ltrim", col, trim) else: - return _invoke_function_over_columns( - "try_make_timestamp_ltz", years, months, days, hours, mins, secs - ) - - -@overload -def make_timestamp_ntz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... - - -@overload -def make_timestamp_ntz( - *, - date: "ColumnOrName", - time: "ColumnOrName", -) -> Column: ... + return _invoke_function_over_columns("ltrim", col) @_try_remote_functions -def make_timestamp_ntz( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, -) -> Column: +def rtrim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: """ - Create local date-time from years, months, days, hours, mins, secs fields. Alternatively, try to - create local date-time from date and time fields. If the configuration `spark.sql.ansi.enabled` - is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error. + Trim the spaces from right end for the specified string value. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or column name, optional + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. + + .. versionadded:: 4.0.0 Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a local date-time. - Returns a column that evaluates to a timestamp. + right trimmed values. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.try_make_timestamp_ntz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.trim` + :meth:`pyspark.sql.functions.ltrim` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Trim the spaces - Example 1: Make local date-time from years, months, days, hours, mins, secs. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select("*", sf.rtrim("value")).show() + +--------+------------+ + | value|rtrim(value)| + +--------+------------+ + | Spark| Spark| + | Spark | Spark| + | Spark| Spark| + +--------+------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +----------------------------------------------------+ - |make_timestamp_ntz(year, month, day, hour, min, sec)| - +----------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +----------------------------------------------------+ + Example 2: Trim specified characters - Example 2: Make local date-time from date and time. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") + >>> df.select("*", sf.rtrim("value", sf.lit("*"))).show() + +--------+---------------------------+ + | value|TRIM(TRAILING * FROM value)| + +--------+---------------------------+ + |***Spark| ***Spark| + | Spark**| Spark| + | *Spark| *Spark| + +--------+---------------------------+ - >>> import pyspark.sql.functions as sf - >>> from datetime import date, time - >>> df = spark.range(1).select( - ... sf.lit(date(2014, 12, 28)).alias("date"), - ... sf.lit(time(6, 30, 45, 887000)).alias("time") - ... ) - >>> df.select(sf.make_timestamp_ntz(date=df.date, time=df.time)).show(truncate=False) - +------------------------------+ - |make_timestamp_ntz(date, time)| - +------------------------------+ - |2014-12-28 06:30:45.887 | - +------------------------------+ + Example 3: Trim a column containing different characters - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) + >>> df.select("*", sf.rtrim("value", "t")).show() + +--------+---+---------------------------+ + | value| t|TRIM(TRAILING t FROM value)| + +--------+---+---------------------------+ + |**Spark*| *| **Spark| + |==Spark=| =| ==Spark| + +--------+---+---------------------------+ """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "make_timestamp_ntz", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) + if trim is not None: + return _invoke_function_over_columns("rtrim", col, trim) else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) - - -@overload -def try_make_timestamp_ntz( - years: "ColumnOrName", - months: "ColumnOrName", - days: "ColumnOrName", - hours: "ColumnOrName", - mins: "ColumnOrName", - secs: "ColumnOrName", -) -> Column: ... - - -@overload -def try_make_timestamp_ntz( - *, - date: "ColumnOrName", - time: "ColumnOrName", -) -> Column: ... + return _invoke_function_over_columns("rtrim", col) @_try_remote_functions -def try_make_timestamp_ntz( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, - days: Optional["ColumnOrName"] = None, - hours: Optional["ColumnOrName"] = None, - mins: Optional["ColumnOrName"] = None, - secs: Optional["ColumnOrName"] = None, - date: Optional["ColumnOrName"] = None, - time: Optional["ColumnOrName"] = None, -) -> Column: +def trim(col: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: """ - Try to create local date-time from years, months, days, hours, mins, secs fields. Alternatively, - try to create local date-time from date and time fields. The function returns NULL on invalid - inputs. + Trim the spaces from both ends for the specified string column. - .. versionadded:: 4.0.0 + .. versionadded:: 1.5.0 - .. versionchanged:: 4.1.0 - Added support for creating timestamps from date and time. + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The year to represent, from 1 to 9999. - Required when creating timestamps from individual components. - Must be used with months, days, hours, mins, and secs. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The month-of-year to represent, from 1 (January) to 12 (December). - Required when creating timestamps from individual components. - Must be used with years, days, hours, mins, and secs. - A column that evaluates to an integer. - days : :class:`~pyspark.sql.Column` or column name, optional - The day-of-month to represent, from 1 to 31. - Required when creating timestamps from individual components. - Must be used with years, months, hours, mins, and secs. - A column that evaluates to an integer. - hours : :class:`~pyspark.sql.Column` or column name, optional - The hour-of-day to represent, from 0 to 23. - Required when creating timestamps from individual components. - Must be used with years, months, days, mins, and secs. - A column that evaluates to an integer. - mins : :class:`~pyspark.sql.Column` or column name, optional - The minute-of-hour to represent, from 0 to 59. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and secs. - A column that evaluates to an integer. - secs : :class:`~pyspark.sql.Column` or column name, optional - The second-of-minute and its micro-fraction to represent, from 0 to 60. - The value can be either an integer like 13, or a fraction like 13.123. - If the sec argument equals to 60, the seconds field is set - to 0 and 1 minute is added to the final timestamp. - Required when creating timestamps from individual components. - Must be used with years, months, days, hours, and mins. - A column that evaluates to a decimal. - date : :class:`~pyspark.sql.Column` or column name, optional - The date to represent, in valid DATE format. - Required when creating timestamps from date and time components. - Must be used with time parameter only. - A column that evaluates to a date. - time : :class:`~pyspark.sql.Column` or column name, optional - The time to represent, in valid TIME format. - Required when creating timestamps from date and time components. - Must be used with date parameter only. - A column that evaluates to a time. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or column name, optional + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. + + .. versionadded:: 4.0.0 Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a local date-time, or NULL in case of an error. - Returns a column that evaluates to a timestamp. + trimmed values from both sides. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.make_timestamp` - :meth:`pyspark.sql.functions.make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_timestamp_ntz` - :meth:`pyspark.sql.functions.try_make_timestamp` - :meth:`pyspark.sql.functions.try_make_timestamp_ltz` - :meth:`pyspark.sql.functions.make_time` - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.ltrim` + :meth:`pyspark.sql.functions.rtrim` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Trim the spaces - Example 1: Make local date-time from years, months, days, hours, mins, secs. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([" Spark", "Spark ", " Spark"], "STRING") + >>> df.select("*", sf.trim("value")).show() + +--------+-----------+ + | value|trim(value)| + +--------+-----------+ + | Spark| Spark| + | Spark | Spark| + | Spark| Spark| + +--------+-----------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ntz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |2014-12-28 06:30:45.887 | - +--------------------------------------------------------+ + Example 2: Trim specified characters - Example 2: Make local date-time with invalid input + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["***Spark", "Spark**", "*Spark"], "STRING") + >>> df.select("*", sf.trim("value", sf.lit("*"))).show() + +--------+-----------------------+ + | value|TRIM(BOTH * FROM value)| + +--------+-----------------------+ + |***Spark| Spark| + | Spark**| Spark| + | *Spark| Spark| + +--------+-----------------------+ - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887]], - ... ['year', 'month', 'day', 'hour', 'min', 'sec']) - >>> df.select( - ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |try_make_timestamp_ntz(year, month, day, hour, min, sec)| - +--------------------------------------------------------+ - |NULL | - +--------------------------------------------------------+ + Example 3: Trim a column containing different characters - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("**Spark*", "*"), ("==Spark=", "=")], ["value", "t"]) + >>> df.select("*", sf.trim("value", "t")).show() + +--------+---+-----------------------+ + | value| t|TRIM(BOTH t FROM value)| + +--------+---+-----------------------+ + |**Spark*| *| Spark| + |==Spark=| =| Spark| + +--------+---+-----------------------+ """ - if years is not None: - if any(arg is not None for arg in [date, time]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "try_make_timestamp_ntz", - _ensure_column_or_name(years), - _ensure_column_or_name(months), - _ensure_column_or_name(days), - _ensure_column_or_name(hours), - _ensure_column_or_name(mins), - _ensure_column_or_name(secs), - ) + if trim is not None: + return _invoke_function_over_columns("trim", col, trim) else: - if any(arg is not None for arg in [years, months, days, hours, mins, secs]): - raise PySparkValueError( - errorClass="CANNOT_SET_TOGETHER", - messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, - ) - return _invoke_function_over_columns( - "try_make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) - ) + return _invoke_function_over_columns("trim", col) @_try_remote_functions -def make_ym_interval( - years: Optional["ColumnOrName"] = None, - months: Optional["ColumnOrName"] = None, -) -> Column: +def concat_ws(sep: str, *cols: "ColumnOrName") -> Column: """ - Make year-month interval from years, months. + Concatenates multiple input string columns together into a single string column, + using the given separator. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - years : :class:`~pyspark.sql.Column` or column name, optional - The number of years, positive or negative. - A column that evaluates to an integer. - months : :class:`~pyspark.sql.Column` or column name, optional - The number of months, positive or negative. - A column that evaluates to an integer. + sep : literal string + words separator. + A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or column name + list of columns to work on. + Each a column that evaluates to a string or an array of strings. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a year-month interval. - Returns a column that evaluates to an interval. + string of concatenated words. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.make_interval` - :meth:`pyspark.sql.functions.make_dt_interval` - :meth:`pyspark.sql.functions.try_make_interval` + :meth:`pyspark.sql.functions.concat` Examples -------- - >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - - Example 1: Make year-month interval from years, months. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) - >>> df.select('*', sf.make_ym_interval('year', df.month)).show(truncate=False) - +----+-----+-------------------------------+ - |year|month|make_ym_interval(year, month) | - +----+-----+-------------------------------+ - |2014|12 |INTERVAL '2015-0' YEAR TO MONTH| - +----+-----+-------------------------------+ - - Example 2: Make year-month interval from years. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) - >>> df.select('*', sf.make_ym_interval(df.year)).show(truncate=False) - +----+-----+-------------------------------+ - |year|month|make_ym_interval(year, 0) | - +----+-----+-------------------------------+ - |2014|12 |INTERVAL '2014-0' YEAR TO MONTH| - +----+-----+-------------------------------+ - - Example 3: Make empty interval. - - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.make_ym_interval()).show(truncate=False) - +----------------------------+ - |make_ym_interval(0, 0) | - +----------------------------+ - |INTERVAL '0-0' YEAR TO MONTH| - +----------------------------+ - - >>> spark.conf.unset("spark.sql.session.timeZone") + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("abcd", "123")], ["s", "d"]) + >>> df.select("*", sf.concat_ws("-", df.s, "d", sf.lit("xyz"))).show() + +----+---+-----------------------+ + | s| d|concat_ws(-, s, d, xyz)| + +----+---+-----------------------+ + |abcd|123| abcd-123-xyz| + +----+---+-----------------------+ """ - _years = lit(0) if years is None else years - _months = lit(0) if months is None else months - return _invoke_function_over_columns("make_ym_interval", _years, _months) - + from pyspark.sql.classic.column import _to_java_column, _to_seq -# ---------------------- Hash Functions ---------------------- + sc = _get_active_spark_context() + return _invoke_function("concat_ws", _enum_to_value(sep), _to_seq(sc, cols, _to_java_column)) @_try_remote_functions -def crc32(col: "ColumnOrName") -> Column: +def decode(col: "ColumnOrName", charset: str) -> Column: """ - Calculates the cyclic redundancy check value (CRC32) of a binary column and - returns the value as a bigint. + Computes the first argument into a string from a binary using the provided character set + (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). + + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -15481,34 +15818,40 @@ def crc32(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. + target column to work on. + charset : literal string + charset to use to decode to. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. - Returns a column that evaluates to a long. - .. versionadded:: 1.5.0 + See Also + -------- + :meth:`pyspark.sql.functions.encode` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC',)], ['a']) - >>> df.select('*', sf.crc32('a')).show(truncate=False) - +---+----------+ - |a |crc32(a) | - +---+----------+ - |ABC|2743272264| - +---+----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(b"\x61\x62\x63\x64",)], ["a"]) + >>> df.select("*", sf.decode("a", "UTF-8")).show() + +-------------+----------------+ + | a|decode(a, UTF-8)| + +-------------+----------------+ + |[61 62 63 64]| abcd| + +-------------+----------------+ """ - return _invoke_function_over_columns("crc32", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("decode", _to_java_column(col), _enum_to_value(charset)) @_try_remote_functions -def md5(col: "ColumnOrName") -> Column: - """Calculates the MD5 digest and returns the value as a 32 character hex string. +def encode(col: "ColumnOrName", charset: str) -> Column: + """ + Computes the first argument into a binary from a string using the provided character set + (one of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). .. versionadded:: 1.5.0 @@ -15518,637 +15861,738 @@ def md5(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. + target column to work on. + A column that evaluates to a string. + charset : literal string + charset to use to encode. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` the column for computed results. - Returns a column that evaluates to a string. + Returns a column that evaluates to a binary. + + See Also + -------- + :meth:`pyspark.sql.functions.decode` Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC',)], ['a']) - >>> df.select('*', sf.md5('a')).show(truncate=False) - +---+--------------------------------+ - |a |md5(a) | - +---+--------------------------------+ - |ABC|902fbdd2b1df0c4f70b4a5d23525e932| - +---+--------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("abcd",)], ["c"]) + >>> df.select("*", sf.encode("c", "UTF-8")).show() + +----+----------------+ + | c|encode(c, UTF-8)| + +----+----------------+ + |abcd| [61 62 63 64]| + +----+----------------+ """ - return _invoke_function_over_columns("md5", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("encode", _to_java_column(col), _enum_to_value(charset)) @_try_remote_functions -def xxh3_64(col: "ColumnOrName") -> Column: - """Returns a 64-bit hash value of the argument using the XXH3 algorithm. - - Unlike :func:`xxhash64`, which hashes one or more columns structurally, this hashes the raw - bytes of a single value with seed 0, so its result is byte compatible with the reference XXH3. +def is_valid_utf8(str: "ColumnOrName") -> Column: + """ + Returns true if the input is a valid UTF-8 string, otherwise returns false. - .. versionadded:: 4.4.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column to hash, which must have string or binary type. + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - Returns a column that evaluates to a long. + whether the input string is a valid UTF-8 string. + Returns a column that evaluates to a boolean. See Also -------- - :meth:`pyspark.sql.functions.xxh3_128` - :meth:`pyspark.sql.functions.xxhash64` + :meth:`pyspark.sql.functions.make_valid_utf8` + :meth:`pyspark.sql.functions.validate_utf8` + :meth:`pyspark.sql.functions.try_validate_utf8` Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark',)], ['a']) - >>> df.select(sf.xxh3_64('a').alias('h')).collect() - [Row(h=80997306238743657)] + >>> spark.range(1).select(sf.is_valid_utf8(sf.lit("SparkSQL"))).show() + +-----------------------+ + |is_valid_utf8(SparkSQL)| + +-----------------------+ + | true| + +-----------------------+ """ - return _invoke_function_over_columns("xxh3_64", col) + return _invoke_function_over_columns("is_valid_utf8", str) @_try_remote_functions -def xxh3_128(col: "ColumnOrName") -> Column: - """Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. +def make_valid_utf8(str: "ColumnOrName") -> Column: + """ + Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the + Unicode replacement character (U+FFFD). - .. versionadded:: 4.4.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column to hash, which must have string or binary type. + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` + the valid UTF-8 version of the given input string. Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.xxh3_64` - :meth:`pyspark.sql.functions.md5` + :meth:`pyspark.sql.functions.is_valid_utf8` + :meth:`pyspark.sql.functions.validate_utf8` + :meth:`pyspark.sql.functions.try_validate_utf8` Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark',)], ['a']) - >>> df.select(sf.xxh3_128('a').alias('h')).collect() - [Row(h='7d57dd84c60c86ca1f4e82ab91a12b5e')] + >>> spark.range(1).select(sf.make_valid_utf8(sf.lit("SparkSQL"))).show() + +-------------------------+ + |make_valid_utf8(SparkSQL)| + +-------------------------+ + | SparkSQL| + +-------------------------+ """ - return _invoke_function_over_columns("xxh3_128", col) + return _invoke_function_over_columns("make_valid_utf8", str) @_try_remote_functions -def sha1(col: "ColumnOrName") -> Column: - """Returns the hex string result of SHA-1. - - .. versionadded:: 1.5.0 +def validate_utf8(str: "ColumnOrName") -> Column: + """ + Returns the input value if it corresponds to a valid UTF-8 string, or emits an error otherwise. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + the input string if it is a valid UTF-8 string, error otherwise. Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.sha` - :meth:`pyspark.sql.functions.sha2` + :meth:`pyspark.sql.functions.is_valid_utf8` + :meth:`pyspark.sql.functions.make_valid_utf8` + :meth:`pyspark.sql.functions.try_validate_utf8` Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC',)], ['a']) - >>> df.select('*', sf.sha1('a')).show(truncate=False) - +---+----------------------------------------+ - |a |sha1(a) | - +---+----------------------------------------+ - |ABC|3c01bdbb26f358bab27f267924aa2c9a03fcfdb8| - +---+----------------------------------------+ + >>> spark.range(1).select(sf.validate_utf8(sf.lit("SparkSQL"))).show() + +-----------------------+ + |validate_utf8(SparkSQL)| + +-----------------------+ + | SparkSQL| + +-----------------------+ """ - return _invoke_function_over_columns("sha1", col) + return _invoke_function_over_columns("validate_utf8", str) @_try_remote_functions -def sha2(col: "ColumnOrName", numBits: int) -> Column: - """Returns the hex string result of SHA-2 family of hash functions (SHA-224, SHA-256, SHA-384, - and SHA-512). The numBits indicates the desired bit length of the result, which must have a - value of 224, 256, 384, 512, or 0 (which is equivalent to 256). - - .. versionadded:: 1.5.0 +def try_validate_utf8(str: "ColumnOrName") -> Column: + """ + Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a binary. - numBits : int - the desired bit length of the result, which must have a - value of 224, 256, 384, 512, or 0 (which is equivalent to 256). - A column that evaluates to an integer. + str : :class:`~pyspark.sql.Column` or column name + A column of strings, each representing a UTF-8 byte sequence. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + the input string if it is a valid UTF-8 string, null otherwise. Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.sha` - :meth:`pyspark.sql.functions.sha1` + :meth:`pyspark.sql.functions.is_valid_utf8` + :meth:`pyspark.sql.functions.make_valid_utf8` + :meth:`pyspark.sql.functions.validate_utf8` Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([['Alice'], ['Bob']], ['name']) - >>> df.select('*', sf.sha2('name', 256)).show(truncate=False) - +-----+----------------------------------------------------------------+ - |name |sha2(name, 256) | - +-----+----------------------------------------------------------------+ - |Alice|3bc51062973c458d5a6f2d8d64a023246354ad7e064b1e4e009ec8a0699a3043| - |Bob |cd9fb1e148ccd8442e5aa74904cc73bf6fb54d1d54d333bd596aa9bb4bb4e961| - +-----+----------------------------------------------------------------+ + >>> spark.range(1).select(sf.try_validate_utf8(sf.lit("SparkSQL"))).show() + +---------------------------+ + |try_validate_utf8(SparkSQL)| + +---------------------------+ + | SparkSQL| + +---------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if numBits not in [0, 224, 256, 384, 512]: - raise PySparkValueError( - errorClass="VALUE_NOT_ALLOWED", - messageParameters={ - "arg_name": "numBits", - "allowed_values": "[0, 224, 256, 384, 512]", - }, - ) - return _invoke_function("sha2", _to_java_column(col), numBits) + return _invoke_function_over_columns("try_validate_utf8", str) @_try_remote_functions -def hash(*cols: "ColumnOrName") -> Column: - """Calculates the hash code of given columns, and returns the result as an int column. - - .. versionadded:: 2.0.0 +def normalize(str: "ColumnOrName", form: Optional["ColumnOrName"] = None) -> Column: + """ + Returns the Unicode normalization of ``str`` using the given normalization ``form``, as + defined by Unicode Standard Annex #15. Normalization is backed by Spark's bundled ICU4J + library rather than the JVM's own Unicode data, so results are stable across JVM vendors + and versions. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.4.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - one or more columns to compute on. - Each a column of any type. + str : :class:`~pyspark.sql.Column` or column name + the input string to normalize. + form : :class:`~pyspark.sql.Column` or column name, optional + the normalization form, one of 'NFC', 'NFD', 'NFKC', 'NFKD' (case-insensitive). + If omitted, 'NFC' is used. Returns ------- :class:`~pyspark.sql.Column` - hash value as int column. - Returns a column that evaluates to an integer. - - See Also - -------- - :meth:`pyspark.sql.functions.xxhash64` + the normalized string. Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) - >>> df.select('*', sf.hash('c1')).show() - +---+---+----------+ - | c1| c2| hash(c1)| - +---+---+----------+ - |ABC|DEF|-757602832| - +---+---+----------+ - - >>> df.select('*', sf.hash('c1', df.c2)).show() - +---+---+------------+ - | c1| c2|hash(c1, c2)| - +---+---+------------+ - |ABC|DEF| 599895104| - +---+---+------------+ - - >>> df.select('*', sf.hash('*')).show() - +---+---+------------+ - | c1| c2|hash(c1, c2)| - +---+---+------------+ - |ABC|DEF| 599895104| - +---+---+------------+ + >>> df = spark.createDataFrame([("\ufb01",)], ["s"]) + >>> df.select(sf.normalize(df.s, sf.lit("NFKC"))).show() + +------------------+ + |normalize(s, NFKC)| + +------------------+ + | fi| + +------------------+ """ - return _invoke_function_over_seq_of_columns("hash", cols) + if form is None: + return _invoke_function_over_columns("normalize", str) + else: + return _invoke_function_over_columns("normalize", str, form) @_try_remote_functions -def xxhash64(*cols: "ColumnOrName") -> Column: - """Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, - and returns the result as a long column. The hash computation uses an initial seed of 42. +def format_number(col: "ColumnOrName", d: int) -> Column: + """ + Formats the number X to a format like '#,--#,--#.--', rounded to d decimal places + with HALF_EVEN round mode, and returns the result as a string. - .. versionadded:: 3.0.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - one or more columns to compute on. - Each a column of any type. + col : :class:`~pyspark.sql.Column` or column name + the column name of the numeric value to be formatted. + A column that evaluates to a numeric. + d : int + the N decimal places. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - hash value as long column. - Returns a column that evaluates to a long. - - See Also - -------- - :meth:`pyspark.sql.functions.hash` - :meth:`pyspark.sql.functions.xxh3_64` + the column of formatted results. + Returns a column that evaluates to a string. Examples -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('ABC', 'DEF')], ['c1', 'c2']) - >>> df.select('*', sf.xxhash64('c1')).show() - +---+---+-------------------+ - | c1| c2| xxhash64(c1)| - +---+---+-------------------+ - |ABC|DEF|4105715581806190027| - +---+---+-------------------+ - - >>> df.select('*', sf.xxhash64('c1', df.c2)).show() - +---+---+-------------------+ - | c1| c2| xxhash64(c1, c2)| - +---+---+-------------------+ - |ABC|DEF|3233247871021311208| - +---+---+-------------------+ - - >>> df.select('*', sf.xxhash64('*')).show() - +---+---+-------------------+ - | c1| c2| xxhash64(c1, c2)| - +---+---+-------------------+ - |ABC|DEF|3233247871021311208| - +---+---+-------------------+ + >>> df = spark.createDataFrame([(5,)], ["a"]) + >>> df.select("*", sf.format_number("a", 4), sf.format_number(df.a, 6)).show() + +---+-------------------+-------------------+ + | a|format_number(a, 4)|format_number(a, 6)| + +---+-------------------+-------------------+ + | 5| 5.0000| 5.000000| + +---+-------------------+-------------------+ """ - return _invoke_function_over_seq_of_columns("xxhash64", cols) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("format_number", _to_java_column(col), _enum_to_value(d)) @_try_remote_functions -def sha(col: "ColumnOrName") -> Column: +def format_string(format: str, *cols: "ColumnOrName") -> Column: """ - Returns a sha1 hash value as a hex string of the `col`. + Formats the arguments in printf-style and returns the result as a string column. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a binary. + format : literal string + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or column name + column names or :class:`~pyspark.sql.Column`\\s to be used in formatting + Each a column of any type. + + Returns + ------- + :class:`~pyspark.sql.Column` + the column of formatted results. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.sha1` - :meth:`pyspark.sql.functions.sha2` + :meth:`pyspark.sql.functions.printf` Examples -------- >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.sha(sf.lit("Spark"))).show() - +--------------------+ - | sha(Spark)| - +--------------------+ - |85f5955f4b27a9a4c...| - +--------------------+ + >>> df = spark.createDataFrame([(5, "hello")], ["a", "b"]) + >>> df.select("*", sf.format_string('%d %s', "a", df.b)).show() + +---+-----+--------------------------+ + | a| b|format_string(%d %s, a, b)| + +---+-----+--------------------------+ + | 5|hello| 5 hello| + +---+-----+--------------------------+ """ - return _invoke_function_over_columns("sha", col) - + from pyspark.sql.classic.column import _to_java_column, _to_seq -# ---------------------- Collection Functions ---------------------- + sc = _get_active_spark_context() + return _invoke_function( + "format_string", _enum_to_value(format), _to_seq(sc, cols, _to_java_column) + ) @_try_remote_functions -def concat(*cols: "ColumnOrName") -> Column: +def instr( + str: "ColumnOrName", + substr: Union[Column, str], + start: Optional[Union[Column, int]] = None, + occurrence: Optional[Union[Column, int]] = None, +) -> Column: """ - Collection function: Concatenates multiple input columns together into a single column. - The function works with strings, numeric, binary and compatible array columns. + Locate the position of the specified occurrence of substr column in the given string. + Returns null if either of the arguments are null. .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.3.0 + Supports optional `start` and `occurrence` parameters. + + Notes + ----- + The position is not zero based, but 1 based index. Returns 0 if substr + could not be found in str. + Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - target column or columns to work on. - Each a column that evaluates to a string, numeric, binary, or array. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + substr : :class:`~pyspark.sql.Column` or literal string + substring to look for. + A column that evaluates to a string. + + .. versionchanged:: 4.0.0 + `substr` now accepts column. + start : int or :class:`~pyspark.sql.Column`, optional + Starting position (1-based, can be negative for backward search). + If not specified, defaults to 1. + A column that evaluates to an integer. + occurrence : int or :class:`~pyspark.sql.Column`, optional + Which occurrence to locate (must be > 0). Defaults to 1. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - concatenated values. Type of the `Column` depends on input columns' type. - Returns a column of the same type as the input. + location of the substring as integer. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.concat_ws` - :meth:`pyspark.sql.functions.array_join` : to concatenate string columns with delimiter + :meth:`pyspark.sql.functions.locate` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.functions.substring_index` Examples -------- - Example 1: Concatenating string columns - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd']) - >>> df.select(sf.concat(df.s, df.d)).show() - +------------+ - |concat(s, d)| - +------------+ - | abcd123| - +------------+ - - Example 2: Concatenating array columns + Example 1: Using a literal string as the 'substring' >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2], [3, 4], [5]), ([1, 2], None, [3])], ['a', 'b', 'c']) - >>> df.select(sf.concat(df.a, df.b, df.c)).show() - +---------------+ - |concat(a, b, c)| - +---------------+ - |[1, 2, 3, 4, 5]| - | NULL| - +---------------+ + >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr(df.s, "b")).show() + +----+-----------+ + | s|instr(s, b)| + +----+-----------+ + |abcd| 2| + | xyz| 0| + +----+-----------+ - Example 3: Concatenating numeric columns + Example 2: Using a Column 'substring' >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) - >>> df.select(sf.concat(df.a, df.b, df.c)).show() - +---------------+ - |concat(a, b, c)| - +---------------+ - | 123| - +---------------+ + >>> df = spark.createDataFrame([("abcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr("s", sf.lit("abc").substr(0, 2))).show() + +----+---------------------------+ + | s|instr(s, substr(abc, 0, 2))| + +----+---------------------------+ + |abcd| 1| + | xyz| 0| + +----+---------------------------+ - Example 4: Concatenating binary columns + Example 3: Using start and occurrence parameters >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytearray(b'abc'), bytearray(b'def'))], ['a', 'b']) - >>> df.select(sf.concat(df.a, df.b)).show() - +-------------------+ - | concat(a, b)| - +-------------------+ - |[61 62 63 64 65 66]| - +-------------------+ + >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr("s", "b", 1, 2)).show() + +-----+-----------------+ + | s|instr(s, b, 1, 2)| + +-----+-----------------+ + |aabcd| 0| + | xyz| 0| + +-----+-----------------+ - Example 5: Concatenating mixed types of columns + Example 4: Using start parameter >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,"abc",3,"def")], ['a','b','c','d']) - >>> df.select(sf.concat(df.a, df.b, df.c, df.d)).show() - +------------------+ - |concat(a, b, c, d)| - +------------------+ - | 1abc3def| - +------------------+ + >>> df = spark.createDataFrame([("aabcd",), ("xyz",)], ["s",]) + >>> df.select("*", sf.instr("s", "a", 2)).show() + +-----+-----------------+ + | s|instr(s, a, 2, 1)| + +-----+-----------------+ + |aabcd| 2| + | xyz| 0| + +-----+-----------------+ """ - return _invoke_function_over_seq_of_columns("concat", cols) + if start is None and occurrence is None: + return _invoke_function_over_columns("instr", str, lit(substr)) + elif start is not None and occurrence is None: + start = lit(start) + return _invoke_function_over_columns("instr", str, lit(substr), start) + else: + start = lit(start) if start is not None else lit(1) + occurrence = lit(occurrence) + return _invoke_function_over_columns("instr", str, lit(substr), start, occurrence) @_try_remote_functions -def element_at(col: "ColumnOrName", extraction: Any) -> Column: +def overlay( + src: "ColumnOrName", + replace: "ColumnOrName", + pos: Union["ColumnOrName", int], + len: Union["ColumnOrName", int] = -1, +) -> Column: """ - Collection function: - (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will - throw an error. If index < 0, accesses elements from the last to the first. - If 'spark.sql.ansi.enabled' is set to true, an exception will be thrown if the index is out - of array boundaries instead of returning NULL. - - (map, key) - Returns value for given key in `extraction` if col is map. The function always - returns NULL if the key is not contained in the map. + Overlay the specified portion of `src` with `replace`, + starting from byte position `pos` of `src` and proceeding for `len` bytes. - .. versionadded:: 2.4.0 + .. versionadded:: 3.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing array or map. - A column that evaluates to an array or map. - extraction : - index to check for in array or key to check for in map. - A column that evaluates to an integer for an array, or the key type for a map. + src : :class:`~pyspark.sql.Column` or column name + the string that will be replaced. + A column that evaluates to a string or binary. + replace : :class:`~pyspark.sql.Column` or column name + the substitution string. + A column that evaluates to a string or binary. + pos : :class:`~pyspark.sql.Column` or column name or int + the starting position in src. + A column that evaluates to an integer. + len : :class:`~pyspark.sql.Column` or column name or int, optional + the number of bytes to replace in src + string by 'replace' defaults to -1, which represents the length of the 'replace' string. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - value at given position. - Returns a column of the element type of the input array, or the value type of the input map. - - Notes - ----- - The position is not zero based, but 1 based index. - If extraction is a string, :meth:`element_at` treats it as a literal string, - while :meth:`try_element_at` treats it as a column name. - - See Also - -------- - :meth:`pyspark.sql.functions.get` - :meth:`pyspark.sql.functions.try_element_at` + string with replaced values. + Returns a column of the same type as the input. Examples -------- - Example 1: Getting the first element of an array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.element_at(df.data, 1)).show() - +-------------------+ - |element_at(data, 1)| - +-------------------+ - | a| - +-------------------+ - - Example 2: Getting the last element of an array using negative index - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.element_at(df.data, -1)).show() - +--------------------+ - |element_at(data, -1)| - +--------------------+ - | c| - +--------------------+ - - Example 3: Getting a value from a map using a key - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.element_at(df.data, sf.lit("a"))).show() - +-------------------+ - |element_at(data, a)| - +-------------------+ - | 1.0| - +-------------------+ + >>> df = spark.createDataFrame([("SPARK_SQL", "CORE")], ("x", "y")) + >>> df.select("*", sf.overlay("x", df.y, 7)).show() + +---------+----+--------------------+ + | x| y|overlay(x, y, 7, -1)| + +---------+----+--------------------+ + |SPARK_SQL|CORE| SPARK_CORE| + +---------+----+--------------------+ - Example 4: Getting a non-existing value from a map using a key + >>> df.select("*", sf.overlay("x", df.y, 7, 0)).show() + +---------+----+-------------------+ + | x| y|overlay(x, y, 7, 0)| + +---------+----+-------------------+ + |SPARK_SQL|CORE| SPARK_CORESQL| + +---------+----+-------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.element_at(df.data, sf.lit("c"))).show() - +-------------------+ - |element_at(data, c)| - +-------------------+ - | NULL| - +-------------------+ + >>> df.select("*", sf.overlay("x", "y", 7, 2)).show() + +---------+----+-------------------+ + | x| y|overlay(x, y, 7, 2)| + +---------+----+-------------------+ + |SPARK_SQL|CORE| SPARK_COREL| + +---------+----+-------------------+ + """ + pos = _enum_to_value(pos) + if not isinstance(pos, (int, str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column, int or str", + "arg_name": "pos", + "arg_type": type(pos).__name__, + }, + ) + len = _enum_to_value(len) + if len is not None and not isinstance(len, (int, str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column, int or str", + "arg_name": "len", + "arg_type": type(len).__name__, + }, + ) - Example 5: Getting a value from a map using a literal string as the key + if isinstance(pos, int): + pos = lit(pos) + if isinstance(len, int): + len = lit(len) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) - >>> df.select(sf.element_at(df.data, 'b')).show() - +-------------------+ - |element_at(data, b)| - +-------------------+ - | 2.0| - +-------------------+ - """ - return _invoke_function_over_columns("element_at", col, lit(extraction)) + return _invoke_function_over_columns("overlay", src, replace, pos, len) @_try_remote_functions -def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: +def sentences( + string: "ColumnOrName", + language: Optional["ColumnOrName"] = None, + country: Optional["ColumnOrName"] = None, +) -> Column: """ - Collection function: - (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will - throw an error. If index < 0, accesses elements from the last to the first. The function - always returns NULL if the index exceeds the length of the array. + Splits a string into arrays of sentences, where each sentence is an array of words. + The `language` and `country` arguments are optional, + When they are omitted: + 1.If they are both omitted, the `Locale.ROOT - locale(language='', country='')` is used. + The `Locale.ROOT` is regarded as the base locale of all locales, and is used as the + language/country neutral locale for the locale sensitive operations. + 2.If the `country` is omitted, the `locale(language, country='')` is used. + When they are null: + 1.If they are both `null`, the `Locale.US - locale(language='en', country='US')` is used. + 2.If the `language` is null and the `country` is not null, + the `Locale.US - locale(language='en', country='US')` is used. + 3.If the `language` is not null and the `country` is null, the `locale(language)` is used. + 4.If neither is `null`, the `locale(language, country)` is used. - (map, key) - Returns value for given key. The function always returns NULL if the key is not - contained in the map. + .. versionadded:: 3.2.0 - .. versionadded:: 3.5.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. versionchanged:: 4.0.0 + Supports `sentences(string, language)`. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing array or map. - A column that evaluates to an array or map. - extraction : - index to check for in array or key to check for in map. - A column that evaluates to an integer for an array, or the key type for a map. + string : :class:`~pyspark.sql.Column` or column name + a string to be split. + A column that evaluates to a string. + language : :class:`~pyspark.sql.Column` or column name, optional + a language of the locale. + A column that evaluates to a string. + country : :class:`~pyspark.sql.Column` or column name, optional + a country of the locale. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - Returns a column of the element type of the input array, or the value type of the input map. - - Notes - ----- - The position is not zero based, but 1 based index. - If extraction is a string, :meth:`try_element_at` treats it as a column name, - while :meth:`element_at` treats it as a literal string. + arrays of split sentences. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.get` - :meth:`pyspark.sql.functions.element_at` + :meth:`pyspark.sql.functions.split` + :meth:`pyspark.sql.functions.split_part` Examples -------- - Example 1: Getting the first element of an array - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit(1))).show() - +-----------------------+ - |try_element_at(data, 1)| - +-----------------------+ - | a| - +-----------------------+ + >>> df = spark.createDataFrame([("This is an example sentence.", )], ["s"]) + >>> df.select("*", sf.sentences(df.s, sf.lit("en"), sf.lit("US"))).show(truncate=False) + +----------------------------+-----------------------------------+ + |s |sentences(s, en, US) | + +----------------------------+-----------------------------------+ + |This is an example sentence.|[[This, is, an, example, sentence]]| + +----------------------------+-----------------------------------+ - Example 2: Getting the last element of an array using negative index + >>> df.select("*", sf.sentences(df.s, sf.lit("en"))).show(truncate=False) + +----------------------------+-----------------------------------+ + |s |sentences(s, en, ) | + +----------------------------+-----------------------------------+ + |This is an example sentence.|[[This, is, an, example, sentence]]| + +----------------------------+-----------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit(-1))).show() - +------------------------+ - |try_element_at(data, -1)| - +------------------------+ - | c| - +------------------------+ + >>> df.select("*", sf.sentences(df.s)).show(truncate=False) + +----------------------------+-----------------------------------+ + |s |sentences(s, , ) | + +----------------------------+-----------------------------------+ + |This is an example sentence.|[[This, is, an, example, sentence]]| + +----------------------------+-----------------------------------+ + """ + if language is None: + language = lit("") + if country is None: + country = lit("") - Example 3: Getting a value from a map using a key + return _invoke_function_over_columns("sentences", string, language, country) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit("a"))).show() - +-----------------------+ - |try_element_at(data, a)| - +-----------------------+ - | 1.0| - +-----------------------+ - Example 4: Getting a non-existing element from an array +@_try_remote_functions +def substring( + str: "ColumnOrName", + pos: Union["ColumnOrName", int], + len: Union["ColumnOrName", int], +) -> Column: + """ + Substring starts at `pos` and is of length `len` when str is String type or + returns the slice of byte array that starts at `pos` in byte and is of length `len` + when str is Binary type. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit(4))).show() - +-----------------------+ - |try_element_at(data, 4)| - +-----------------------+ - | NULL| - +-----------------------+ + .. versionadded:: 1.5.0 - Example 5: Getting a non-existing value from a map using a key + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) - >>> df.select(sf.try_element_at(df.data, sf.lit("c"))).show() - +-----------------------+ - |try_element_at(data, c)| - +-----------------------+ - | NULL| - +-----------------------+ + Notes + ----- + The position is not zero based, but 1 based index. - Example 6: Getting a value from a map using a column name as the key + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string or binary. + pos : :class:`~pyspark.sql.Column` or column name or int + starting position in str. + A column that evaluates to an integer. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) - >>> df.select(sf.try_element_at(df.data, 'b')).show() - +-----------------------+ - |try_element_at(data, b)| - +-----------------------+ - | 1.0| - +-----------------------+ + .. versionchanged:: 4.0.0 + `pos` now accepts column and column name. + + len : :class:`~pyspark.sql.Column` or column name or int + length of chars. + A column that evaluates to an integer. + + .. versionchanged:: 4.0.0 + `len` now accepts column and column name. + + Returns + ------- + :class:`~pyspark.sql.Column` + substring of given value. + Returns a column of the same type as the input. + + See Also + -------- + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.locate` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.Column.substr` + + Examples + -------- + Example 1: Using literal integers as arguments + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('abcd',)], ['s',]) + >>> df.select('*', sf.substring(df.s, 1, 2)).show() + +----+------------------+ + | s|substring(s, 1, 2)| + +----+------------------+ + |abcd| ab| + +----+------------------+ + + Example 2: Using columns as arguments + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) + >>> df.select('*', sf.substring(df.s, 2, df.l)).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, 2, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + >>> df.select('*', sf.substring(df.s, df.p, 3)).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, p, 3)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + >>> df.select('*', sf.substring(df.s, df.p, df.l)).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, p, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + Example 3: Using column names as arguments + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('Spark', 2, 3)], ['s', 'p', 'l']) + >>> df.select('*', sf.substring(df.s, 2, 'l')).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, 2, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ + + >>> df.select('*', sf.substring('s', 'p', 'l')).show() + +-----+---+---+------------------+ + | s| p| l|substring(s, p, l)| + +-----+---+---+------------------+ + |Spark| 2| 3| par| + +-----+---+---+------------------+ """ - return _invoke_function_over_columns("try_element_at", col, extraction) + pos = _enum_to_value(pos) + pos = lit(pos) if isinstance(pos, int) else pos + len = _enum_to_value(len) + len = lit(len) if isinstance(len, int) else len + return _invoke_function_over_columns("substring", str, pos, len) @_try_remote_functions -def size(col: "ColumnOrName") -> Column: +def substring_index(str: "ColumnOrName", delim: str, count: int) -> Column: """ - Collection function: returns the length of the array or map stored in the column. + Returns the substring from string str before count occurrences of the delimiter delim. + If count is positive, everything the left of the final delimiter (counting from left) is + returned. If count is negative, every to the right of the final delimiter (counting from the + right) is returned. substring_index performs a case-sensitive match when searching for delim. .. versionadded:: 1.5.0 @@ -16157,309 +16601,302 @@ def size(col: "ColumnOrName") -> Column: Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array or map. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + delim : literal string + delimiter of values. + A column that evaluates to a string. + count : int + number of occurrences. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - length of the array/map. - Returns a column that evaluates to an integer. + substring of given value. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.locate` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.Column.substr` Examples -------- - >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data']) - >>> df.select(size(df.data)).collect() - [Row(size(data)=3), Row(size(data)=1), Row(size(data)=0)] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('a.b.c.d',)], ['s']) + >>> df.select('*', sf.substring_index(df.s, '.', 2)).show() + +-------+------------------------+ + | s|substring_index(s, ., 2)| + +-------+------------------------+ + |a.b.c.d| a.b| + +-------+------------------------+ + + >>> df.select('*', sf.substring_index('s', '.', -3)).show() + +-------+-------------------------+ + | s|substring_index(s, ., -3)| + +-------+-------------------------+ + |a.b.c.d| b.c.d| + +-------+-------------------------+ """ - return _invoke_function_over_columns("size", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "substring_index", _to_java_column(str), _enum_to_value(delim), _enum_to_value(count) + ) @_try_remote_functions -def cardinality(col: "ColumnOrName") -> Column: - """ - Collection function: returns the length of the array or map stored in the column. +def levenshtein( + left: "ColumnOrName", right: "ColumnOrName", threshold: Optional[int] = None +) -> Column: + """Computes the Levenshtein distance of the two given strings. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to an array or map. + left : :class:`~pyspark.sql.Column` or column name + first column value. + A column that evaluates to a string. + right : :class:`~pyspark.sql.Column` or column name + second column value. + A column that evaluates to a string. + threshold : int, optional + if set when the levenshtein distance of the two given strings + less than or equal to a given threshold then return result distance, or -1. + A column that evaluates to an integer. + + .. versionadded:: 3.5.0 Returns ------- :class:`~pyspark.sql.Column` - length of the array/map. + Levenshtein distance as integer value. Returns a column that evaluates to an integer. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [([1, 2, 3],),([1],),([],)], ['data'] - ... ).select(sf.cardinality("data")).show() - +-----------------+ - |cardinality(data)| - +-----------------+ - | 3| - | 1| - | 0| - +-----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('kitten', 'sitting',)], ['l', 'r']) + >>> df.select('*', sf.levenshtein('l', 'r')).show() + +------+-------+-----------------+ + | l| r|levenshtein(l, r)| + +------+-------+-----------------+ + |kitten|sitting| 3| + +------+-------+-----------------+ + + >>> df.select('*', sf.levenshtein(df.l, df.r, 2)).show() + +------+-------+--------------------+ + | l| r|levenshtein(l, r, 2)| + +------+-------+--------------------+ + |kitten|sitting| -1| + +------+-------+--------------------+ """ - return _invoke_function_over_columns("cardinality", col) + from pyspark.sql.classic.column import _to_java_column + + if threshold is None: + return _invoke_function_over_columns("levenshtein", left, right) + else: + return _invoke_function( + "levenshtein", _to_java_column(left), _to_java_column(right), _enum_to_value(threshold) + ) @_try_remote_functions -def array_sort( - col: "ColumnOrName", comparator: Optional[Callable[[Column, Column], Column]] = None -) -> Column: - """ - Collection function: sorts the input array in ascending order. The elements of the input array - must be orderable. Null elements will be placed at the end of the returned array. - - .. versionadded:: 2.4.0 +def jaro_winkler_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Computes the Jaro-Winkler similarity between the two given strings. - .. versionchanged:: 3.4.0 - Can take a `comparator` function. + The result is a double between 0.0 (no similarity) and 1.0 (identical strings). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - comparator : callable, optional - A binary ``(Column, Column) -> Column: ...``. - The comparator will take two - arguments representing two elements of the array. It returns a negative integer, 0, or a - positive integer as the first element is less than, equal to, or greater than the second - element. If the comparator function returns null, the function will fail and raise an error. + left : :class:`~pyspark.sql.Column` or column name + first column value. + A column that evaluates to a string. + right : :class:`~pyspark.sql.Column` or column name + second column value. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - sorted array. - Returns a column that evaluates to an array. - - See Also - -------- - :meth:`pyspark.sql.functions.sort_array` + Jaro-Winkler similarity as a double value. + Returns a column that evaluates to a double. Examples -------- - >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data']) - >>> df.select(array_sort(df.data).alias('r')).collect() - [Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])] - >>> df = spark.createDataFrame([(["foo", "foobar", None, "bar"],),(["foo"],),([],)], ['data']) - >>> df.select(array_sort( - ... "data", - ... lambda x, y: when(x.isNull() | y.isNull(), lit(0)).otherwise(length(y) - length(x)) - ... ).alias("r")).collect() - [Row(r=['foobar', 'foo', None, 'bar']), Row(r=['foo']), Row(r=[])] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('MARTHA', 'MARHTA')], ['l', 'r']) + >>> df.select(sf.jaro_winkler_similarity('l', 'r')).show() + +-----------------------------+ + |jaro_winkler_similarity(l, r)| + +-----------------------------+ + | 0.9611111111111111| + +-----------------------------+ """ - if comparator is None: - return _invoke_function_over_columns("array_sort", col) - else: - return _invoke_higher_order_function("array_sort", [col], [comparator]) + return _invoke_function_over_columns("jaro_winkler_similarity", left, right) @_try_remote_functions -def reverse(col: "ColumnOrName") -> Column: +def locate(substr: str, str: "ColumnOrName", pos: int = 1) -> Column: """ - Collection function: returns a reversed string, a binary value with bytes in reverse order, - or an array with elements in reverse order. + Locate the position of the first occurrence of substr in a string column, after position pos. .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - .. versionchanged:: 4.2.0 - Added support for binary type. - Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the element to be reversed. - A column that evaluates to a string, binary, or array. + substr : literal string + a string. + A column that evaluates to a string. + str : :class:`~pyspark.sql.Column` or column name + a Column of :class:`pyspark.sql.types.StringType`. + A column that evaluates to a string. + pos : int, optional + start position (zero based). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a reversed string, a binary value with bytes in reverse order, - or an array with elements in reverse order. - Returns a column of the same type as the input. - - Examples - -------- - Example 1: Reverse a string + position of the substring. + Returns a column that evaluates to an integer. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('Spark SQL',)], ['data']) - >>> df.select(sf.reverse(df.data)).show() - +-------------+ - |reverse(data)| - +-------------+ - | LQS krapS| - +-------------+ + Notes + ----- + The position is not zero based, but 1 based index. Returns 0 if substr + could not be found in str. - Example 2: Reverse an array + See Also + -------- + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.substr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.Column.substr` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],) ,([1],) ,([],)], ['data']) - >>> df.select(sf.reverse(df.data)).show() - +-------------+ - |reverse(data)| - +-------------+ - | [3, 1, 2]| - | [1]| - | []| - +-------------+ - - Example 3: Reverse binary data + >>> df = spark.createDataFrame([('abcd',)], ['s',]) + >>> df.select('*', sf.locate('b', 's', 1)).show() + +----+---------------+ + | s|locate(b, s, 1)| + +----+---------------+ + |abcd| 2| + +----+---------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytearray(b"\\xCA\\xFE"),)], "data: binary") - >>> df.select(sf.hex(sf.reverse(df.data))).show() - +------------------+ - |hex(reverse(data))| - +------------------+ - | FECA| - +------------------+ + >>> df.select('*', sf.locate('b', df.s, 3)).show() + +----+---------------+ + | s|locate(b, s, 3)| + +----+---------------+ + |abcd| 0| + +----+---------------+ """ - return _invoke_function_over_columns("reverse", col) + from pyspark.sql.classic.column import _to_java_column + return _invoke_function( + "locate", _enum_to_value(substr), _to_java_column(str), _enum_to_value(pos) + ) -def _unresolved_named_lambda_variable(name: str) -> Column: + +@_try_remote_functions +def lpad( + col: "ColumnOrName", + len: Union[Column, int], + pad: Union[Column, str], +) -> Column: """ - Create `o.a.s.sql.expressions.UnresolvedNamedLambdaVariable`, - convert it to o.s.sql.Column and wrap in Python `Column` + Left-pad the string column to width `len` with `pad`. + + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - name_parts : str - """ - from py4j.java_gateway import JVMView + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string or binary. + len : :class:`~pyspark.sql.Column` or int + length of the final string. + A column that evaluates to an integer. - sc = _get_active_spark_context() - return Column(cast(JVMView, sc._jvm).PythonSQLUtils.unresolvedNamedLambdaVariable(name)) + .. versionchanged:: 4.0.0 + `pattern` now accepts column. + pad : :class:`~pyspark.sql.Column` or literal string + chars to prepend. + A column that evaluates to a string or binary. -def _get_lambda_parameters(f: Callable) -> ValuesView[inspect.Parameter]: - signature = inspect.signature(f) - parameters = signature.parameters.values() + .. versionchanged:: 4.0.0 + `pattern` now accepts column. - # We should exclude functions that use - # variable args and keyword argnames - # as well as keyword only args - supported_parameter_types = { - inspect.Parameter.POSITIONAL_OR_KEYWORD, - inspect.Parameter.POSITIONAL_ONLY, - } + Returns + ------- + :class:`~pyspark.sql.Column` + left padded result. + Returns a column of the same type as the input. - # Validate that - # function arity is between 1 and 3 - if not (1 <= len(parameters) <= 3): - raise PySparkValueError( - errorClass="WRONG_NUM_ARGS_FOR_HIGHER_ORDER_FUNCTION", - messageParameters={"func_name": f.__name__, "num_args": str(len(parameters))}, - ) - - # and all arguments can be used as positional - if not all(p.kind in supported_parameter_types for p in parameters): - raise PySparkValueError( - errorClass="UNSUPPORTED_PARAM_TYPE_FOR_HIGHER_ORDER_FUNCTION", - messageParameters={"func_name": f.__name__}, - ) - - return parameters - - -def _create_lambda(f: Callable) -> Callable: - """ - Create `o.a.s.sql.expressions.LambdaFunction` corresponding - to transformation described by f - - :param f: A Python of one of the following forms: - - (Column) -> Column: ... - - (Column, Column) -> Column: ... - - (Column, Column, Column) -> Column: ... - """ - from py4j.java_gateway import JVMView - - from pyspark.sql.classic.column import _to_seq - - parameters = _get_lambda_parameters(f) - - sc = _get_active_spark_context() - - argnames = ["x", "y", "z"] - args = [_unresolved_named_lambda_variable(arg) for arg in argnames[: len(parameters)]] - - result = f(*args) - - if not isinstance(result, Column): - raise PySparkValueError( - errorClass="HIGHER_ORDER_FUNCTION_SHOULD_RETURN_COLUMN", - messageParameters={"func_name": f.__name__, "return_type": type(result).__name__}, - ) - - jexpr = result._jc - jargs = _to_seq(sc, [arg._jc for arg in args]) - return cast(JVMView, sc._jvm).PythonSQLUtils.lambdaFunction(jexpr, jargs) + See Also + -------- + :meth:`pyspark.sql.functions.rpad` + Examples + -------- + Example 1: Pad with a literal string -def _invoke_higher_order_function( - name: str, - cols: Sequence["ColumnOrName"], - funs: Sequence[Callable], -) -> Column: - """ - Invokes expression identified by name, - (relative to ```org.apache.spark.sql.catalyst.expressions``) - and wraps the result with Column (first Scala one, then Python). + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.lpad(df.s, 6, '#')).show() + +----+-------------+ + | s|lpad(s, 6, #)| + +----+-------------+ + |abcd| ##abcd| + | xyz| ###xyz| + | 12| ####12| + +----+-------------+ - :param name: Name of the expression - :param cols: a list of columns - :param funs: a list of (*Column) -> Column functions. + Example 2: Pad with a bytes column - :return: a Column + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.lpad(df.s, 6, sf.lit(b"\x75\x76"))).show() + +----+-------------------+ + | s|lpad(s, 6, X'7576')| + +----+-------------------+ + |abcd| uvabcd| + | xyz| uvuxyz| + | 12| uvuv12| + +----+-------------------+ """ - from py4j.java_gateway import JVMView - - from pyspark.sql.classic.column import _to_java_column, _to_seq - - sc = _get_active_spark_context() - jfuns = [_create_lambda(f) for f in funs] - jcols = [_to_java_column(c) for c in cols] - return Column(cast(JVMView, sc._jvm).PythonSQLUtils.fn(name, _to_seq(sc, jcols + jfuns))) - - -@overload -def transform(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... - - -@overload -def transform(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... + return _invoke_function_over_columns("lpad", col, lit(len), lit(pad)) @_try_remote_functions -def transform( +def rpad( col: "ColumnOrName", - f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], + len: Union[Column, int], + pad: Union[Column, str], ) -> Column: """ - Returns an array of elements after applying a transformation to each element in the input array. + Right-pad the string column to width `len` with `pad`. - .. versionadded:: 3.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -16467,3181 +16904,3083 @@ def transform( Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - a function that is applied to each element of the input array. - Can take one of the following forms: + target column to work on. + A column that evaluates to a string or binary. + len : :class:`~pyspark.sql.Column` or int + length of the final string. + A column that evaluates to an integer. - - Unary ``(x: Column) -> Column: ...`` - - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is - a 0-based index of the element. + .. versionchanged:: 4.0.0 + `pattern` now accepts column. - and can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + pad : :class:`~pyspark.sql.Column` or literal string + chars to prepend. + A column that evaluates to a string or binary. + + .. versionchanged:: 4.0.0 + `pattern` now accepts column. Returns ------- :class:`~pyspark.sql.Column` - a new array of transformed elements. - Returns a column that evaluates to an array. + right padded result. + Returns a column of the same type as the input. + + See Also + -------- + :meth:`pyspark.sql.functions.lpad` Examples -------- - >>> df = spark.createDataFrame([(1, [1, 2, 3, 4])], ("key", "values")) - >>> df.select(transform("values", lambda x: x * 2).alias("doubled")).show() - +------------+ - | doubled| - +------------+ - |[2, 4, 6, 8]| - +------------+ + Example 1: Pad with a literal string - >>> def alternate(x, i): - ... return when(i % 2 == 0, x).otherwise(-x) - ... - >>> df.select(transform("values", alternate).alias("alternated")).show() - +--------------+ - | alternated| - +--------------+ - |[1, -2, 3, -4]| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.rpad(df.s, 6, '#')).show() + +----+-------------+ + | s|rpad(s, 6, #)| + +----+-------------+ + |abcd| abcd##| + | xyz| xyz###| + | 12| 12####| + +----+-------------+ + + Example 2: Pad with a bytes column + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('abcd',), ('xyz',), ('12',)], ['s',]) + >>> df.select("*", sf.rpad(df.s, 6, sf.lit(b"\x75\x76"))).show() + +----+-------------------+ + | s|rpad(s, 6, X'7576')| + +----+-------------------+ + |abcd| abcduv| + | xyz| xyzuvu| + | 12| 12uvuv| + +----+-------------------+ """ - return _invoke_higher_order_function("transform", [col], [f]) + return _invoke_function_over_columns("rpad", col, lit(len), lit(pad)) @_try_remote_functions -def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: +def repeat(col: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: """ - Returns whether a predicate holds for one or more elements in the array. + Repeats a string column n times, and returns it as a new string column. - .. versionadded:: 3.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - ``(x: Column) -> Column: ...`` returning the Boolean expression. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + n : :class:`~pyspark.sql.Column` or column name or int + number of times to repeat value. + A column that evaluates to an integer. + + .. versionchanged:: 4.0.0 + `n` now accepts column and column name. Returns ------- :class:`~pyspark.sql.Column` - True if "any" element of an array evaluates to True when passed as an argument to - given function and False otherwise. - Returns a column that evaluates to a boolean. + string with repeated values. + Returns a column that evaluates to a string. Examples -------- - >>> df = spark.createDataFrame([(1, [1, 2, 3, 4]), (2, [3, -1, 0])],("key", "values")) - >>> df.select(exists("values", lambda x: x < 0).alias("any_negative")).show() - +------------+ - |any_negative| - +------------+ - | false| - | true| - +------------+ + Example 1: Repeat with a constant number of times + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ab',)], ['s',]) + >>> df.select("*", sf.repeat("s", 3)).show() + +---+------------+ + | s|repeat(s, 3)| + +---+------------+ + | ab| ababab| + +---+------------+ + + >>> df.select("*", sf.repeat(df.s, sf.lit(4))).show() + +---+------------+ + | s|repeat(s, 4)| + +---+------------+ + | ab| abababab| + +---+------------+ + + Example 2: Repeat with a column containing different number of times + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ab', 5,), ('abc', 6,)], ['s', 't']) + >>> df.select("*", sf.repeat("s", "t")).show() + +---+---+------------------+ + | s| t| repeat(s, t)| + +---+---+------------------+ + | ab| 5| ababababab| + |abc| 6|abcabcabcabcabcabc| + +---+---+------------------+ """ - return _invoke_higher_order_function("exists", [col], [f]) + n = _enum_to_value(n) + n = lit(n) if isinstance(n, int) else n + return _invoke_function_over_columns("repeat", col, n) @_try_remote_functions -def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: +def split( + str: "ColumnOrName", + pattern: Union[Column, str], + limit: Union["ColumnOrName", int] = -1, +) -> Column: """ - Returns whether a predicate holds for every element in the array. + Splits str around matches of the given pattern. - .. versionadded:: 3.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - ``(x: Column) -> Column: ...`` returning the Boolean expression. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + str : :class:`~pyspark.sql.Column` or column name + a string expression to split. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or literal string + a string representing a regular expression. The regex string should be + a Java regular expression. + A column that evaluates to a string. + + .. versionchanged:: 4.0.0 + `pattern` now accepts column. Does not accept column name since string type remain + accepted as a regular expression representation, for backwards compatibility. + In addition to int, `limit` now accepts column and column name. + + limit : :class:`~pyspark.sql.Column` or column name or int + an integer which controls the number of times `pattern` is applied. + A column that evaluates to an integer. + + * ``limit > 0``: The resulting array's length will not be more than `limit`, and the + resulting array's last entry will contain all input beyond the last + matched pattern. + * ``limit <= 0``: `pattern` will be applied as many times as possible, and the resulting + array can be of any size. + + .. versionchanged:: 3.0 + `split` now takes an optional `limit` field. If not provided, default limit value is -1. Returns ------- :class:`~pyspark.sql.Column` - True if "all" elements of an array evaluates to True when passed as an argument to - given function and False otherwise. - Returns a column that evaluates to a boolean. + array of separated strings. + Returns a column that evaluates to an array. + + See Also + -------- + :meth:`pyspark.sql.functions.sentences` + :meth:`pyspark.sql.functions.split_part` Examples -------- - >>> df = spark.createDataFrame( - ... [(1, ["bar"]), (2, ["foo", "bar"]), (3, ["foobar", "foo"])], - ... ("key", "values") - ... ) - >>> df.select(forall("values", lambda x: x.rlike("foo")).alias("all_foo")).show() - +-------+ - |all_foo| - +-------+ - | false| - | false| - | true| - +-------+ - """ - return _invoke_higher_order_function("forall", [col], [f]) + Example 1: Repeat with a constant pattern + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('oneAtwoBthreeC',)], ['s',]) + >>> df.select('*', sf.split(df.s, '[ABC]')).show() + +--------------+-------------------+ + | s|split(s, [ABC], -1)| + +--------------+-------------------+ + |oneAtwoBthreeC|[one, two, three, ]| + +--------------+-------------------+ -@overload -def filter(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... + >>> df.select('*', sf.split(df.s, '[ABC]', 2)).show() + +--------------+------------------+ + | s|split(s, [ABC], 2)| + +--------------+------------------+ + |oneAtwoBthreeC| [one, twoBthreeC]| + +--------------+------------------+ + >>> df.select('*', sf.split('s', '[ABC]', -2)).show() + +--------------+-------------------+ + | s|split(s, [ABC], -2)| + +--------------+-------------------+ + |oneAtwoBthreeC|[one, two, three, ]| + +--------------+-------------------+ -@overload -def filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... + Example 2: Repeat with a column containing different patterns and limits + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([ + ... ('oneAtwoBthreeC', '[ABC]', 2), + ... ('1A2B3C', '[1-9]+', 1), + ... ('aa2bb3cc4', '[1-9]+', -1)], ['s', 'p', 'l']) + >>> df.select('*', sf.split(df.s, df.p)).show() + +--------------+------+---+-------------------+ + | s| p| l| split(s, p, -1)| + +--------------+------+---+-------------------+ + |oneAtwoBthreeC| [ABC]| 2|[one, two, three, ]| + | 1A2B3C|[1-9]+| 1| [, A, B, C]| + | aa2bb3cc4|[1-9]+| -1| [aa, bb, cc, ]| + +--------------+------+---+-------------------+ -@_try_remote_functions -def filter( - col: "ColumnOrName", - f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], -) -> Column: + >>> df.select(sf.split('s', df.p, 'l')).show() + +-----------------+ + | split(s, p, l)| + +-----------------+ + |[one, twoBthreeC]| + | [1A2B3C]| + | [aa, bb, cc, ]| + +-----------------+ """ - Returns an array of elements for which a predicate holds in a given array. + limit = _enum_to_value(limit) + limit = lit(limit) if isinstance(limit, int) else limit + return _invoke_function_over_columns("split", str, lit(pattern), limit) - .. versionadded:: 3.1.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def rlike(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. + + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - f : function - A function that returns the Boolean expression. - Can take one of the following forms: - - - Unary ``(x: Column) -> Column: ...`` - - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is - a 0-based index of the element. - - and can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - filtered array of elements where given function evaluated to True - when passed as an argument. - Returns a column that evaluates to an array. + true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. + + See Also + -------- + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.regexp_like` + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.ilike` Examples -------- - >>> df = spark.createDataFrame( - ... [(1, ["2018-09-20", "2019-02-03", "2019-07-01", "2020-06-01"])], - ... ("key", "values") - ... ) - >>> def after_second_quarter(x): - ... return month(to_date(x)) > 6 - ... - >>> df.select( - ... filter("values", after_second_quarter).alias("after_second_quarter") - ... ).show(truncate=False) - +------------------------+ - |after_second_quarter | - +------------------------+ - |[2018-09-20, 2019-07-01]| - +------------------------+ - """ - return _invoke_higher_order_function("filter", [col], [f]) + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("1a 2b 14m", r"(\d+)")], ["str", "regexp"]) + >>> df.select('*', sf.rlike('str', sf.lit(r'(\d+)'))).show() + +---------+------+-----------------+ + | str|regexp|RLIKE(str, (\d+))| + +---------+------+-----------------+ + |1a 2b 14m| (\d+)| true| + +---------+------+-----------------+ + >>> df.select('*', sf.rlike('str', sf.lit(r'\d{2}b'))).show() + +---------+------+------------------+ + | str|regexp|RLIKE(str, \d{2}b)| + +---------+------+------------------+ + |1a 2b 14m| (\d+)| false| + +---------+------+------------------+ -@_try_remote_functions -def aggregate( - col: "ColumnOrName", - initialValue: "ColumnOrName", - merge: Callable[[Column, Column], Column], - finish: Optional[Callable[[Column], Column]] = None, -) -> Column: + >>> df.select('*', sf.rlike("str", sf.col("regexp"))).show() + +---------+------+------------------+ + | str|regexp|RLIKE(str, regexp)| + +---------+------+------------------+ + |1a 2b 14m| (\d+)| true| + +---------+------+------------------+ + + >>> df.select('*', sf.rlike("str", "regexp")).show() + +---------+------+------------------+ + | str|regexp|RLIKE(str, regexp)| + +---------+------+------------------+ + |1a 2b 14m| (\d+)| true| + +---------+------+------------------+ """ - Applies a binary operator to an initial state and all elements in the array, - and reduces this to a single state. The final state is converted into the final result - by applying a finish function. + return _invoke_function_over_columns("rlike", str, regexp) - Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). - .. versionadded:: 3.1.0 +@_try_remote_functions +def regexp(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression. - A column of any type. - merge : function - a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``initialValue``. - finish : function, optional - an optional unary function ``(x: Column) -> Column: ...`` - used to convert accumulated value. + str : :class:`~pyspark.sql.Column` or str + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or str + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - final value after aggregate function is applied. - Returns a column of the same type as ``initialValue``. + true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. + + See Also + -------- + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp_like` + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.ilike` Examples -------- - >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) - >>> df.select(aggregate("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() - +----+ - | sum| - +----+ - |42.0| - +----+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp('str', sf.lit(r'(\d+)'))).show() + +------------------+ + |REGEXP(str, (\d+))| + +------------------+ + | true| + +------------------+ - >>> def merge(acc, x): - ... count = acc.count + 1 - ... sum = acc.sum + x - ... return struct(count.alias("count"), sum.alias("sum")) - ... - >>> df.select( - ... aggregate( - ... "values", - ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), - ... merge, - ... lambda acc: acc.sum / acc.count, - ... ).alias("mean") - ... ).show() - +----+ - |mean| - +----+ - | 8.4| - +----+ - """ - if finish is not None: - return _invoke_higher_order_function("aggregate", [col, initialValue], [merge, finish]) + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp('str', sf.lit(r'\d{2}b'))).show() + +-------------------+ + |REGEXP(str, \d{2}b)| + +-------------------+ + | false| + +-------------------+ - else: - return _invoke_higher_order_function("aggregate", [col, initialValue], [merge]) + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp('str', sf.col("regexp"))).show() + +-------------------+ + |REGEXP(str, regexp)| + +-------------------+ + | true| + +-------------------+ + """ + return _invoke_function_over_columns("regexp", str, regexp) @_try_remote_functions -def reduce( - col: "ColumnOrName", - initialValue: "ColumnOrName", - merge: Callable[[Column, Column], Column], - finish: Optional[Callable[[Column], Column]] = None, -) -> Column: - """ - Applies a binary operator to an initial state and all elements in the array, - and reduces this to a single state. The final state is converted into the final result - by applying a finish function. - - Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). +def regexp_like(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns true if `str` matches the Java regex `regexp`, or false otherwise. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - initialValue : :class:`~pyspark.sql.Column` or str - initial value. Name of column or expression. - A column of any type. - merge : function - a binary function ``(acc: Column, x: Column) -> Column...`` returning expression - of the same type as ``zero``. - finish : function, optional - an optional unary function ``(x: Column) -> Column: ...`` - used to convert accumulated value. + str : :class:`~pyspark.sql.Column` or str + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or str + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - final value after aggregate function is applied. - Returns a column of the same type as ``initialValue``. + true if `str` matches a Java regex, or false otherwise. + Returns a column that evaluates to a boolean. + + See Also + -------- + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.ilike` Examples -------- - >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) - >>> df.select(reduce("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() - +----+ - | sum| - +----+ - |42.0| - +----+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp_like('str', sf.lit(r'(\d+)'))).show() + +-----------------------+ + |REGEXP_LIKE(str, (\d+))| + +-----------------------+ + | true| + +-----------------------+ - >>> def merge(acc, x): - ... count = acc.count + 1 - ... sum = acc.sum + x - ... return struct(count.alias("count"), sum.alias("sum")) - ... - >>> df.select( - ... reduce( - ... "values", - ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), - ... merge, - ... lambda acc: acc.sum / acc.count, - ... ).alias("mean") - ... ).show() - +----+ - |mean| - +----+ - | 8.4| - +----+ - """ - if finish is not None: - return _invoke_higher_order_function("reduce", [col, initialValue], [merge, finish]) + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp_like('str', sf.lit(r'\d{2}b'))).show() + +------------------------+ + |REGEXP_LIKE(str, \d{2}b)| + +------------------------+ + | false| + +------------------------+ - else: - return _invoke_higher_order_function("reduce", [col, initialValue], [merge]) + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("1a 2b 14m", r"(\d+)")], ["str", "regexp"] + ... ).select(sf.regexp_like('str', sf.col("regexp"))).show() + +------------------------+ + |REGEXP_LIKE(str, regexp)| + +------------------------+ + | true| + +------------------------+ + """ + return _invoke_function_over_columns("regexp_like", str, regexp) @_try_remote_functions -def zip_with( - left: "ColumnOrName", - right: "ColumnOrName", - f: Callable[[Column, Column], Column], -) -> Column: - """ - Merge two given arrays, element-wise, into a single array using a function. - If one array is shorter, nulls are appended at the end to match the length of the longer - array, before applying the function. - - .. versionadded:: 3.1.0 +def randstr(length: Union[Column, int], seed: Optional[Union[Column, int]] = None) -> Column: + """Returns a string of the specified length whose characters are chosen uniformly at random from + the following pool of characters: 0-9, a-z, A-Z. The random seed is optional. The string length + must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or str - name of the first column or expression. - A column that evaluates to an array. - right : :class:`~pyspark.sql.Column` or str - name of the second column or expression. - A column that evaluates to an array. - f : function - a binary function ``(x1: Column, x2: Column) -> Column...`` - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + length : :class:`~pyspark.sql.Column` or int + Number of characters in the string to generate. + A column that evaluates to an integer. Must be a constant. + seed : :class:`~pyspark.sql.Column` or int + Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - array of calculated values derived by applying given function to each pair of arguments. - Returns a column that evaluates to an array. + The generated random string with the specified length. + Returns a column that evaluates to a string. - Examples + See Also -------- - >>> df = spark.createDataFrame([(1, [1, 3, 5, 8], [0, 2, 4, 6])], ("id", "xs", "ys")) - >>> df.select(zip_with("xs", "ys", lambda x, y: x ** y).alias("powers")).show(truncate=False) - +---------------------------+ - |powers | - +---------------------------+ - |[1.0, 9.0, 625.0, 262144.0]| - +---------------------------+ + :meth:`pyspark.sql.functions.rand` + :meth:`pyspark.sql.functions.randn` - >>> df = spark.createDataFrame([(1, ["foo", "bar"], [1, 2, 3])], ("id", "xs", "ys")) - >>> df.select(zip_with("xs", "ys", lambda x, y: concat_ws("_", x, y)).alias("xs_ys")).show() - +-----------------+ - | xs_ys| - +-----------------+ - |[foo_1, bar_2, 3]| - +-----------------+ + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(0, 10, 1, 1).select(sf.randstr(16, 3)).show() + +----------------+ + | randstr(16, 3)| + +----------------+ + |nurJIpH4cmmMnsCG| + |fl9YtT5m01trZtIt| + |PD19rAgscTHS7qQZ| + |2CuAICF5UJOruVv4| + |kNZEs8nDpJEoz3Rl| + |OXiU0KN5eaXfjXFs| + |qfnTM1BZAHtN0gBV| + |1p8XiSKwg33KnRPK| + |od5y5MucayQq1bKK| + |tklYPmKmc5sIppWM| + +----------------+ """ - return _invoke_higher_order_function("zip_with", [left, right], [f]) + length = _enum_to_value(length) + length = lit(length) + if seed is None: + return _invoke_function_over_columns("randstr", length) + else: + seed = _enum_to_value(seed) + seed = lit(seed) + return _invoke_function_over_columns("randstr", length, seed) @_try_remote_functions -def transform_keys(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: - """ - Applies a function to every key-value pair in a map and returns - a map with the results of those applications as the new keys for the pairs. - - .. versionadded:: 3.1.0 +def regexp_count(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns a count of the number of times that the Java regex pattern `regexp` is matched + in the string `str`. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression - f : function - a binary function ``(k: Column, v: Column) -> Column...`` - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - a new map of entries where new keys were calculated by applying given function to - each key value argument. + the number of times that a Java regex pattern is matched in the string. + Returns a column that evaluates to an integer. Examples -------- - >>> df = spark.createDataFrame([(1, {"foo": -2.0, "bar": 2.0})], ("id", "data")) - >>> row = df.select(transform_keys( - ... "data", lambda k, _: upper(k)).alias("data_upper") - ... ).head() - >>> sorted(row["data_upper"].items()) - [('BAR', 2.0), ('FOO', -2.0)] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) + >>> df.select('*', sf.regexp_count('str', sf.lit(r'\d+'))).show() + +---------+------+----------------------+ + | str|regexp|regexp_count(str, \d+)| + +---------+------+----------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+----------------------+ + + >>> df.select('*', sf.regexp_count('str', sf.lit(r'mmm'))).show() + +---------+------+----------------------+ + | str|regexp|regexp_count(str, mmm)| + +---------+------+----------------------+ + |1a 2b 14m| \d+| 0| + +---------+------+----------------------+ + + >>> df.select('*', sf.regexp_count("str", sf.col("regexp"))).show() + +---------+------+-------------------------+ + | str|regexp|regexp_count(str, regexp)| + +---------+------+-------------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+-------------------------+ + + >>> df.select('*', sf.regexp_count(sf.col('str'), "regexp")).show() + +---------+------+-------------------------+ + | str|regexp|regexp_count(str, regexp)| + +---------+------+-------------------------+ + |1a 2b 14m| \d+| 3| + +---------+------+-------------------------+ """ - return _invoke_higher_order_function("transform_keys", [col], [f]) + return _invoke_function_over_columns("regexp_count", str, regexp) @_try_remote_functions -def transform_values(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: - """ - Applies a function to every key-value pair in a map and returns - a map with the results of those applications as the new values for the pairs. +def regexp_extract(str: "ColumnOrName", pattern: str, idx: int) -> Column: + r"""Extract a specific group matched by the Java regex `regexp`, from the specified string column. + If the regex did not match, or the specified group did not match, an empty string is returned. - .. versionadded:: 3.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression - f : function - a binary function ``(k: Column, v: Column) -> Column...`` - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + pattern : str + regex pattern to apply. + A column that evaluates to a string. + idx : int + matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - a new map of entries where new values were calculated by applying given function to - each key value argument. + matched value specified by `idx` group id. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.regexp_extract_all` Examples -------- - >>> df = spark.createDataFrame([(1, {"IT": 10.0, "SALES": 2.0, "OPS": 24.0})], ("id", "data")) - >>> row = df.select(transform_values( - ... "data", lambda k, v: when(k.isin("IT", "OPS"), v + 10.0).otherwise(v) - ... ).alias("new_data")).head() - >>> sorted(row["new_data"].items()) - [('IT', 20.0), ('OPS', 34.0), ('SALES', 2.0)] - """ - return _invoke_higher_order_function("transform_values", [col], [f]) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('100-200',)], ['str']) + >>> df.select('*', sf.regexp_extract('str', r'(\d+)-(\d+)', 1)).show() + +-------+-----------------------------------+ + | str|regexp_extract(str, (\d+)-(\d+), 1)| + +-------+-----------------------------------+ + |100-200| 100| + +-------+-----------------------------------+ + >>> df = spark.createDataFrame([('foo',)], ['str']) + >>> df.select('*', sf.regexp_extract('str', r'(\d+)', 1)).show() + +---+-----------------------------+ + |str|regexp_extract(str, (\d+), 1)| + +---+-----------------------------+ + |foo| | + +---+-----------------------------+ -@_try_remote_functions -def map_filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: + >>> df = spark.createDataFrame([('aaaac',)], ['str']) + >>> df.select('*', sf.regexp_extract(sf.col('str'), '(a+)(b)?(c)', 2)).show() + +-----+-----------------------------------+ + | str|regexp_extract(str, (a+)(b)?(c), 2)| + +-----+-----------------------------------+ + |aaaac| | + +-----+-----------------------------------+ """ - Collection function: Returns a new map column whose key-value pairs satisfy a given - predicate function. + from pyspark.sql.classic.column import _to_java_column - .. versionadded:: 3.1.0 + return _invoke_function( + "regexp_extract", _to_java_column(str), _enum_to_value(pattern), _enum_to_value(idx) + ) - .. versionchanged:: 3.4.0 - Supports Spark Connect. + +@_try_remote_functions +def regexp_extract_all( + str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None +) -> Column: + r"""Extract all strings in the `str` that match the Java regex `regexp` + and corresponding to the regex group index. + + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or a column expression representing the map to be filtered. - f : function - A binary function ``(k: Column, v: Column) -> Column...`` that defines the predicate. - This function should return a boolean column that will be used to filter the input map. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. + idx : :class:`~pyspark.sql.Column` or int, optional + matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A new map column containing only the key-value pairs that satisfy the predicate. + all strings in the `str` that match a Java regex and corresponding to the regex group index. + Returns a column that evaluates to an array. - Examples + See Also -------- - Example 1: Filtering a map with a simple condition + :meth:`pyspark.sql.functions.regexp_extract` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) - >>> row = df.select( - ... sf.map_filter("data", lambda _, v: v > 30.0).alias("data_filtered") - ... ).head() - >>> sorted(row["data_filtered"].items()) - [('baz', 32.0), ('foo', 42.0)] + >>> df = spark.createDataFrame([("100-200, 300-400", r"(\d+)-(\d+)")], ["str", "regexp"]) + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'))).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+---------------------------------------+ - Example 2: Filtering a map with a condition on keys + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), sf.lit(1))).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 1)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+---------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) - >>> row = df.select( - ... sf.map_filter("data", lambda k, _: k.startswith("b")).alias("data_filtered") - ... ).head() - >>> sorted(row["data_filtered"].items()) - [('bar', 1.0), ('baz', 32.0)] + >>> df.select('*', sf.regexp_extract_all('str', sf.lit(r'(\d+)-(\d+)'), 2)).show() + +----------------+-----------+---------------------------------------+ + | str| regexp|regexp_extract_all(str, (\d+)-(\d+), 2)| + +----------------+-----------+---------------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [200, 400]| + +----------------+-----------+---------------------------------------+ - Example 3: Filtering a map with a complex condition + >>> df.select('*', sf.regexp_extract_all('str', sf.col("regexp"))).show() + +----------------+-----------+----------------------------------+ + | str| regexp|regexp_extract_all(str, regexp, 1)| + +----------------+-----------+----------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+----------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) - >>> row = df.select( - ... sf.map_filter("data", lambda k, v: k.startswith("b") & (v > 1.0)).alias("data_filtered") - ... ).head() - >>> sorted(row["data_filtered"].items()) - [('baz', 32.0)] + >>> df.select('*', sf.regexp_extract_all(sf.col('str'), "regexp")).show() + +----------------+-----------+----------------------------------+ + | str| regexp|regexp_extract_all(str, regexp, 1)| + +----------------+-----------+----------------------------------+ + |100-200, 300-400|(\d+)-(\d+)| [100, 300]| + +----------------+-----------+----------------------------------+ """ - return _invoke_higher_order_function("map_filter", [col], [f]) + if idx is None: + return _invoke_function_over_columns("regexp_extract_all", str, regexp) + else: + return _invoke_function_over_columns("regexp_extract_all", str, regexp, lit(idx)) @_try_remote_functions -def map_zip_with( - col1: "ColumnOrName", - col2: "ColumnOrName", - f: Callable[[Column, Column, Column], Column], +def regexp_replace( + string: "ColumnOrName", + pattern: Union[str, Column], + replacement: Union[str, Column], + position: Optional[Union[int, Column]] = None, ) -> Column: - """ - Collection: Merges two given maps into a single map by applying a function to - the key-value pairs. + r"""Replace all substrings of the specified string value that match regexp with replacement. - .. versionadded:: 3.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + .. versionchanged:: 4.3.0 + Supports the `position` parameter. Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - The name of the first column or a column expression representing the first map. - col2 : :class:`~pyspark.sql.Column` or str - The name of the second column or a column expression representing the second map. - f : function - A ternary function ``(k: Column, v1: Column, v2: Column) -> Column...`` that defines - how to merge the values from the two maps. This function should return a column that - will be used as the value in the resulting map. - Can use methods of :class:`~pyspark.sql.Column`, functions defined in - :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. - Python ``UserDefinedFunctions`` are not supported - (`SPARK-27052 `__). + string : :class:`~pyspark.sql.Column` or str + column name or column containing the string value. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or str + column object or str containing the regexp pattern. + A column that evaluates to a string. + replacement : :class:`~pyspark.sql.Column` or str + column object or str containing the replacement. + A column that evaluates to a string. + position : :class:`~pyspark.sql.Column` or int, optional + position to start replacement. The first position is 1. + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new map column where each key-value pair is the result of applying the function to - the corresponding key-value pairs in the input maps. + string with all substrings replaced. + Returns a column that evaluates to a string. Examples -------- - Example 1: Merging two maps with a simple function - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], - ... ("id", "map1", "map2")) - >>> row = df.select( - ... sf.map_zip_with("map1", "map2", lambda _, v1, v2: v1 + v2).alias("updated_data") - ... ).head() - >>> sorted(row["updated_data"].items()) - [('A', 4), ('B', 6)] - - Example 2: Merging two maps with a complex function - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], - ... ("id", "map1", "map2")) - >>> row = df.select( - ... sf.map_zip_with("map1", "map2", - ... lambda k, v1, v2: sf.when(k == "A", v1 + v2).otherwise(v1 - v2) - ... ).alias("updated_data") - ... ).head() - >>> sorted(row["updated_data"].items()) - [('A', 4), ('B', -2)] - - Example 3: Merging two maps with mismatched keys - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... (1, {"A": 1, "B": 2}, {"B": 3, "C": 4})], - ... ("id", "map1", "map2")) - >>> row = df.select( - ... sf.map_zip_with("map1", "map2", - ... lambda _, v1, v2: sf.when(v2.isNull(), v1).otherwise(v1 + v2) - ... ).alias("updated_data") - ... ).head() - >>> sorted(row["updated_data"].items()) - [('A', 1), ('B', 5), ('C', None)] - """ - return _invoke_higher_order_function("map_zip_with", [col1, col2], [f]) + >>> df = spark.createDataFrame( + ... [("100-200", r"(\d+)", "--")], + ... ["str", "pattern", "replacement"] + ... ) + Example 1: Replaces all the substrings in the `str` column name that + match the regex pattern `(\d+)` (one or more digits) with the replacement + string "--". -# ---------------------- Array Functions ---------------------- + >>> df.select('*', sf.regexp_replace('str', r'(\d+)', '--')).show() + +-------+-------+-----------+---------------------------------+ + | str|pattern|replacement|regexp_replace(str, (\d+), --, 1)| + +-------+-------+-----------+---------------------------------+ + |100-200| (\d+)| --| -----| + +-------+-------+-----------+---------------------------------+ + Example 2: Replaces all the substrings in the `str` Column that match + the regex pattern in the `pattern` Column with the string in the `replacement` + column. -@overload -def array(*cols: "ColumnOrName") -> Column: ... + >>> df.select('*', \ + ... sf.regexp_replace(sf.col("str"), sf.col("pattern"), sf.col("replacement")) \ + ... ).show() + +-------+-------+-----------+--------------------------------------------+ + | str|pattern|replacement|regexp_replace(str, pattern, replacement, 1)| + +-------+-------+-----------+--------------------------------------------+ + |100-200| (\d+)| --| -----| + +-------+-------+-----------+--------------------------------------------+ + Example 3: Replaces substrings starting from the specified position. + For the input string "100-200", position 5 starts replacement after "100-". -@overload -def array(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... + >>> df.select(sf.regexp_replace("str", r"(\d+)", "--", 5).alias("d")).show() + +------+ + | d| + +------+ + |100---| + +------+ + """ + if position is None: + return _invoke_function_over_columns( + "regexp_replace", string, lit(pattern), lit(replacement) + ) + else: + return _invoke_function_over_columns( + "regexp_replace", + string, + lit(pattern), + lit(replacement), + lit(position), + ) @_try_remote_functions -def array( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], -) -> Column: - """ - Collection function: Creates a new array column from the input columns or column names. - - .. versionadded:: 1.4.0 +def regexp_substr(str: "ColumnOrName", regexp: "ColumnOrName") -> Column: + r"""Returns the first substring that matches the Java regex `regexp` within the string `str`. + If the regular expression is not found, the result is null. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - Column names or :class:`~pyspark.sql.Column` objects that have the same data type. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new Column of array type, where each value is an array containing the corresponding values - from the input columns. - Returns a column that evaluates to an array. + the first substring that matches a Java regex within the string `str`. + Returns a column that evaluates to a string. Examples -------- - Example 1: Basic usage of array function with column names. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array('name', 'occupation')).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, doctor]| - | [Bob, engineer]| - +-----------------------+ + >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+")], ["str", "regexp"]) - Example 2: Usage of array function with Column objects. + Example 1: Returns the first substring in the `str` column name that + matches the regex pattern `(\d+)` (one or more digits). - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array(df.name, df.occupation)).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, doctor]| - | [Bob, engineer]| - +-----------------------+ + >>> df.select('*', sf.regexp_substr('str', sf.lit(r'\d+'))).show() + +---------+------+-----------------------+ + | str|regexp|regexp_substr(str, \d+)| + +---------+------+-----------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+-----------------------+ - Example 3: Single argument as list of column names. + Example 2: Returns the first substring in the `str` column name that + matches the regex pattern `(mmm)` (three consecutive 'm' characters) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array(['name', 'occupation'])).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, doctor]| - | [Bob, engineer]| - +-----------------------+ + >>> df.select('*', sf.regexp_substr('str', sf.lit(r'mmm'))).show() + +---------+------+-----------------------+ + | str|regexp|regexp_substr(str, mmm)| + +---------+------+-----------------------+ + |1a 2b 14m| \d+| NULL| + +---------+------+-----------------------+ - Example 4: Usage of array function with columns of different types. + Example 3: Returns the first substring in the `str` column name that + matches the regex pattern in `regexp` Column. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("Alice", 2, 22.2), ("Bob", 5, 36.1)], - ... ("name", "age", "weight")) - >>> df.select(sf.array(['age', 'weight'])).show() - +------------------+ - |array(age, weight)| - +------------------+ - | [2.0, 22.2]| - | [5.0, 36.1]| - +------------------+ + >>> df.select('*', sf.regexp_substr("str", sf.col("regexp"))).show() + +---------+------+--------------------------+ + | str|regexp|regexp_substr(str, regexp)| + +---------+------+--------------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+--------------------------+ - Example 5: array function with a column containing null values. + Example 4: Returns the first substring in the `str` Column that + matches the regex pattern in `regexp` column name. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", None), ("Bob", "engineer")], - ... ("name", "occupation")) - >>> df.select(sf.array('name', 'occupation')).show() - +-----------------------+ - |array(name, occupation)| - +-----------------------+ - | [Alice, NULL]| - | [Bob, engineer]| - +-----------------------+ + >>> df.select('*', sf.regexp_substr(sf.col("str"), "regexp")).show() + +---------+------+--------------------------+ + | str|regexp|regexp_substr(str, regexp)| + +---------+------+--------------------------+ + |1a 2b 14m| \d+| 1| + +---------+------+--------------------------+ """ - if len(cols) == 1 and isinstance(cols[0], (list, set)): - cols = cols[0] # type: ignore[assignment] - return _invoke_function_over_seq_of_columns("array", cols) # type: ignore[arg-type] + return _invoke_function_over_columns("regexp_substr", str, regexp) @_try_remote_functions -def array_contains(col: "ColumnOrName", value: Any) -> Column: - """ - Collection function: Returns true if the array contains the value, false if not. Returns - null if the array or value is null, or if the value is not found and the array contains a - null element. - - .. versionadded:: 1.5.0 +def regexp_instr( + str: "ColumnOrName", regexp: "ColumnOrName", idx: Optional[Union[int, Column]] = None +) -> Column: + r"""Returns the position of the first substring in the `str` that match the Java regex `regexp` + and corresponding to the regex group index. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The target column containing the arrays. - A column that evaluates to an array. - value : - The value or column to check for in the array. - A column of the same type as the array elements. + str : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. + regexp : :class:`~pyspark.sql.Column` or column name + regex pattern to apply. + A column that evaluates to a string. + idx : :class:`~pyspark.sql.Column` or int, optional + matched group id. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A new Column of Boolean type, where each value indicates whether the corresponding array - from the input column contains the specified value. - Returns a column that evaluates to a boolean. - - See Also - -------- - :meth:`pyspark.sql.functions.array_position` + the position of the first substring in the `str` that match a Java regex and corresponding + to the regex group index. + Returns a column that evaluates to an integer. Examples -------- - Example 1: Basic usage of array_contains function. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],), ([],)], ['data']) - >>> df.select(sf.array_contains(df.data, "a")).show() - +-----------------------+ - |array_contains(data, a)| - +-----------------------+ - | true| - | false| - +-----------------------+ + >>> df = spark.createDataFrame([("1a 2b 14m", r"\d+(a|b|m)")], ["str", "regexp"]) - Example 2: Usage of array_contains function with a column. + Example 1: Returns the position of the first substring in the `str` column name that + match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'). - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], "c"), - ... (["c", "d", "e"], "d"), - ... (["e", "a", "c"], "b")], ["data", "item"]) - >>> df.select(sf.array_contains(df.data, sf.col("item"))).show() - +--------------------------+ - |array_contains(data, item)| - +--------------------------+ - | true| - | true| - | false| - +--------------------------+ + >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'))).show() + +---------+----------+--------------------------------+ + | str| regexp|regexp_instr(str, \d+(a|b|m), 0)| + +---------+----------+--------------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+--------------------------------+ - Example 3: Attempt to use array_contains function with a null array. + Example 2: Returns the position of the first substring in the `str` column name that + match the regex pattern `(\d+(a|b|m))` (one or more digits followed by 'a', 'b', or 'm'), - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), (["a", "b", "c"],)], ['data']) - >>> df.select(sf.array_contains(df.data, "a")).show() - +-----------------------+ - |array_contains(data, a)| - +-----------------------+ - | NULL| - | true| - +-----------------------+ + >>> df.select('*', sf.regexp_instr('str', sf.lit(r'\d+(a|b|m)'), sf.lit(1))).show() + +---------+----------+--------------------------------+ + | str| regexp|regexp_instr(str, \d+(a|b|m), 1)| + +---------+----------+--------------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+--------------------------------+ - Example 4: Usage of array_contains with an array column containing null values. + Example 3: Returns the position of the first substring in the `str` column name that + match the regex pattern in `regexp` Column. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_contains(df.data, "a")).show() - +-----------------------+ - |array_contains(data, a)| - +-----------------------+ - | true| - +-----------------------+ + >>> df.select('*', sf.regexp_instr('str', sf.col("regexp"))).show() + +---------+----------+----------------------------+ + | str| regexp|regexp_instr(str, regexp, 0)| + +---------+----------+----------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+----------------------------+ - Example 5: Value absent from an array that contains a null element returns NULL. + Example 4: Returns the position of the first substring in the `str` Column that + match the regex pattern in `regexp` column name. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_contains(df.data, "b")).show() - +-----------------------+ - |array_contains(data, b)| - +-----------------------+ - | NULL| - +-----------------------+ + >>> df.select('*', sf.regexp_instr(sf.col("str"), "regexp")).show() + +---------+----------+----------------------------+ + | str| regexp|regexp_instr(str, regexp, 0)| + +---------+----------+----------------------------+ + |1a 2b 14m|\d+(a|b|m)| 1| + +---------+----------+----------------------------+ """ - return _invoke_function_over_columns("array_contains", col, lit(value)) + if idx is None: + return _invoke_function_over_columns("regexp_instr", str, regexp) + else: + return _invoke_function_over_columns("regexp_instr", str, regexp, lit(idx)) @_try_remote_functions -def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column: - """ - Collection function: This function returns a boolean column indicating if the input arrays - have common non-null elements, returning true if they do, null if the arrays do not contain - any common elements but are not empty and at least one of them contains a null element, - and false otherwise. +def initcap(col: "ColumnOrName") -> Column: + """Translate the first letter of each word to upper case in the sentence. - .. versionadded:: 2.4.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - a1, a2 : :class:`~pyspark.sql.Column` or str - The names of the columns that contain the input arrays. - Each a column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new Column of Boolean type, where each value indicates whether the corresponding arrays - from the input columns contain any common elements. - Returns a column that evaluates to a boolean. + string with all first letters are uppercase in each word. + Returns a column that evaluates to a string. Examples -------- - Example 1: Basic usage of arrays_overlap function. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ab cd',)], ['a']) + >>> df.select("*", sf.initcap("a")).show() + +-----+----------+ + | a|initcap(a)| + +-----+----------+ + |ab cd| Ab Cd| + +-----+----------+ + """ + return _invoke_function_over_columns("initcap", col) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b"], ["b", "c"]), (["a"], ["b", "c"])], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | true| - | false| - +--------------------+ - Example 2: Usage of arrays_overlap function with arrays containing null elements. +@_try_remote_functions +def soundex(col: "ColumnOrName") -> Column: + """ + Returns the SoundEx encoding for a string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None], ["b", None]), (["a"], ["b", "c"])], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | NULL| - | false| - +--------------------+ + .. versionadded:: 1.5.0 - Example 3: Usage of arrays_overlap function with arrays that are null. + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None, ["b", "c"]), (["a"], None)], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | NULL| - | NULL| - +--------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a string. - Example 4: Usage of arrays_overlap on arrays with identical elements. + Returns + ------- + :class:`~pyspark.sql.Column` + SoundEx encoded string. + Returns a column that evaluates to a string. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b"], ["a", "b"]), (["a"], ["a"])], ['x', 'y']) - >>> df.select(sf.arrays_overlap(df.x, df.y)).show() - +--------------------+ - |arrays_overlap(x, y)| - +--------------------+ - | true| - | true| - +--------------------+ + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("Peters",),("Uhrbach",)], ["s"]) + >>> df.select("*", sf.soundex("s")).show() + +-------+----------+ + | s|soundex(s)| + +-------+----------+ + | Peters| P362| + |Uhrbach| U612| + +-------+----------+ """ - return _invoke_function_over_columns("arrays_overlap", a1, a2) + return _invoke_function_over_columns("soundex", col) @_try_remote_functions -def slice( - x: "ColumnOrName", start: Union["ColumnOrName", int], length: Union["ColumnOrName", int] -) -> Column: - """ - Array function: Returns a new array column by slicing the input array column from - a start index to a specific length. The indices start at 1, and can be negative to index - from the end of the array. The length specifies the number of elements in the resulting array. +def bin(col: "ColumnOrName") -> Column: + """Returns the string representation of the binary value of the given column. - .. versionadded:: 2.4.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - x : :class:`~pyspark.sql.Column` or str - Input array column or column name to be sliced. - A column that evaluates to an array. - start : :class:`~pyspark.sql.Column`, str, or int - The start index for the slice operation. If negative, starts the index from the - end of the array. - A column that evaluates to an integer. - length : :class:`~pyspark.sql.Column`, str, or int - The length of the slice, representing number of elements in the resulting array. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a long. Returns ------- :class:`~pyspark.sql.Column` - A new Column object of Array type, where each value is a slice of the corresponding - list from the input column. - Returns a column that evaluates to an array. + binary representation of given value as string. + Returns a column that evaluates to a string. Examples -------- - Example 1: Basic usage of the slice function. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) - >>> df.select(sf.slice(df.x, 2, 2)).show() - +--------------+ - |slice(x, 2, 2)| - +--------------+ - | [2, 3]| - | [5]| - +--------------+ - - Example 2: Slicing with negative start index. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) - >>> df.select(sf.slice(df.x, -1, 1)).show() - +---------------+ - |slice(x, -1, 1)| - +---------------+ - | [3]| - | [5]| - +---------------+ - - Example 3: Slice function with column inputs for start and length. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3], 2, 2), ([4, 5], 1, 3)], ['x', 'start', 'length']) - >>> df.select(sf.slice(df.x, df.start, df.length)).show() - +-----------------------+ - |slice(x, start, length)| - +-----------------------+ - | [2, 3]| - | [4, 5]| - +-----------------------+ - """ - start = _enum_to_value(start) - start = lit(start) if isinstance(start, int) else start - length = _enum_to_value(length) - length = lit(length) if isinstance(length, int) else length - - return _invoke_function_over_columns("slice", x, start, length) + >>> import pyspark.sql.functions as sf + >>> spark.range(10).select("*", sf.bin("id")).show() + +---+-------+ + | id|bin(id)| + +---+-------+ + | 0| 0| + | 1| 1| + | 2| 10| + | 3| 11| + | 4| 100| + | 5| 101| + | 6| 110| + | 7| 111| + | 8| 1000| + | 9| 1001| + +---+-------+ + """ + return _invoke_function_over_columns("bin", col) @_try_remote_functions -def trim_array(x: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: - """ - Array function: Returns the given array column with the last ``n`` elements removed. - Raises an error if ``n`` is negative or greater than the number of elements in the array. +def hex(col: "ColumnOrName") -> Column: + """Computes hex value of the given column, which could be :class:`pyspark.sql.types.StringType`, + :class:`pyspark.sql.types.BinaryType`, :class:`pyspark.sql.types.IntegerType` or + :class:`pyspark.sql.types.LongType`. - .. versionadded:: 4.4.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - x : :class:`~pyspark.sql.Column` or str - Input array column or column name to be trimmed. - A column that evaluates to an array. - n : :class:`~pyspark.sql.Column`, str, or int - The number of elements to remove from the end of the array. Must be between 0 and - the number of elements in the array (inclusive). - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. + A column that evaluates to a long, binary, or string. + + See Also + -------- + :meth:`pyspark.sql.functions.unhex` Returns ------- :class:`~pyspark.sql.Column` - A new Column object of Array type, where each value is the corresponding input array - with its last ``n`` elements removed. - Returns a column that evaluates to an array. + hexadecimal representation of given value as string. + Returns a column that evaluates to a string. Examples -------- - Example 1: Basic usage of the trim_array function. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 4, 5],), ([4, 5],)], ['x']) - >>> df.select(sf.trim_array(df.x, 2)).show() - +----------------+ - |trim_array(x, 2)| - +----------------+ - | [1, 2, 3]| - | []| - +----------------+ - - Example 2: trim_array function with a column input for n. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 4, 5], 1), ([4, 5], 0)], ['x', 'n']) - >>> df.select(sf.trim_array(df.x, df.n)).show() - +----------------+ - |trim_array(x, n)| - +----------------+ - | [1, 2, 3, 4]| - | [4, 5]| - +----------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('ABC', 3)], ['a', 'b']) + >>> df.select('*', sf.hex('a'), sf.hex(df.b)).show() + +---+---+------+------+ + | a| b|hex(a)|hex(b)| + +---+---+------+------+ + |ABC| 3|414243| 3| + +---+---+------+------+ """ - n = _enum_to_value(n) - n = lit(n) if isinstance(n, int) else n - return _invoke_function_over_columns("trim_array", x, n) + return _invoke_function_over_columns("hex", col) @_try_remote_functions -def array_join( - col: "ColumnOrName", delimiter: str, null_replacement: Optional[str] = None -) -> Column: - """ - Array function: Returns a string column by concatenating the elements of the input - array column using the delimiter. Null values within the array can be replaced with - a specified string through the null_replacement argument. If null_replacement is - not set, null values are ignored. +def unhex(col: "ColumnOrName") -> Column: + """Inverse of hex. Interprets each pair of characters as a hexadecimal number + and converts to the byte representation of number. - .. versionadded:: 2.4.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The input column containing the arrays to be joined. - A column that evaluates to an array. - delimiter : str - The string to be used as the delimiter when joining the array elements. - A column that evaluates to a string. - null_replacement : str, optional - The string to replace null values within the array. If not set, null values are ignored. + col : :class:`~pyspark.sql.Column` or column name + target column to work on. A column that evaluates to a string. + See Also + -------- + :meth:`pyspark.sql.functions.hex` + Returns ------- :class:`~pyspark.sql.Column` - A new column of string type, where each value is the result of joining the corresponding - array from the input column. - Returns a column that evaluates to a string. - - See Also - -------- - :meth:`pyspark.sql.functions.concat` - :meth:`pyspark.sql.functions.concat_ws` + byte representation of the given hexadecimal value. + Returns a column that evaluates to a binary. Examples -------- - Example 1: Basic usage of array_join function. - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],), (["a", "b"],)], ['data']) - >>> df.select(sf.array_join(df.data, ",")).show() - +-------------------+ - |array_join(data, ,)| - +-------------------+ - | a,b,c| - | a,b| - +-------------------+ - - Example 2: Usage of array_join function with null_replacement argument. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('414243',)], ['a']) + >>> df.select('*', sf.unhex('a')).show() + +------+----------+ + | a| unhex(a)| + +------+----------+ + |414243|[41 42 43]| + +------+----------+ + """ + return _invoke_function_over_columns("unhex", col) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_join(df.data, ",", "NULL")).show() - +-------------------------+ - |array_join(data, ,, NULL)| - +-------------------------+ - | a,NULL,c| - +-------------------------+ - Example 3: Usage of array_join function without null_replacement argument. +@_try_remote_functions +def uniform( + min: Union[Column, int, float], + max: Union[Column, int, float], + seed: Optional[Union[Column, int]] = None, +) -> Column: + """Returns a random value with independent and identically distributed (i.i.d.) values with the + specified range of numbers. The random seed is optional. The provided numbers specifying the + minimum and maximum values of the range must be constant. If both of these numbers are integers, + then the result will also be an integer. Otherwise if one or both of these are floating-point + numbers, then the result will also be a floating-point number. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) - >>> df.select(sf.array_join(df.data, ",")).show() - +-------------------+ - |array_join(data, ,)| - +-------------------+ - | a,c| - +-------------------+ + .. versionadded:: 4.0.0 - Example 4: Usage of array_join function with an array that is null. + Parameters + ---------- + min : :class:`~pyspark.sql.Column`, int, or float + Minimum value in the range. + A column that evaluates to a numeric. Must be a constant. + max : :class:`~pyspark.sql.Column`, int, or float + Maximum value in the range. + A column that evaluates to a numeric. Must be a constant. + seed : :class:`~pyspark.sql.Column` or int + Optional random number seed to use. + A column that evaluates to an integer or long. Must be a constant. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([(None,)], schema) - >>> df.select(sf.array_join(df.data, ",")).show() - +-------------------+ - |array_join(data, ,)| - +-------------------+ - | NULL| - +-------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + The generated random number within the specified range. + Returns a column of the same type as the input. - Example 5: Usage of array_join function with an array containing only null values. + See Also + -------- + :meth:`pyspark.sql.functions.rand` + :meth:`pyspark.sql.functions.randn` - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([([None, None],)], schema) - >>> df.select(sf.array_join(df.data, ",", "NULL")).show() - +-------------------------+ - |array_join(data, ,, NULL)| - +-------------------------+ - | NULL,NULL| - +-------------------------+ + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(0, 10, 1, 1).select(sf.uniform(5, 105, 3)).show() + +------------------+ + |uniform(5, 105, 3)| + +------------------+ + | 30| + | 71| + | 99| + | 77| + | 16| + | 25| + | 89| + | 80| + | 51| + | 83| + +------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - _get_active_spark_context() - if null_replacement is None: - return _invoke_function("array_join", _to_java_column(col), _enum_to_value(delimiter)) + min = _enum_to_value(min) + min = lit(min) + max = _enum_to_value(max) + max = lit(max) + if seed is None: + return _invoke_function_over_columns("uniform", min, max) else: - return _invoke_function( - "array_join", - _to_java_column(col), - _enum_to_value(delimiter), - _enum_to_value(null_replacement), - ) + seed = _enum_to_value(seed) + seed = lit(seed) + return _invoke_function_over_columns("uniform", min, max, seed) @_try_remote_functions -def array_position(col: "ColumnOrName", value: Any) -> Column: - """ - Array function: Locates the position of the first occurrence of the given value - in the given array. Returns null if either of the arguments are null. +def length(col: "ColumnOrName") -> Column: + """Computes the character length of string data or number of bytes of binary data. + The length of character data includes the trailing spaces. The length of binary data + includes binary zeros. - .. versionadded:: 2.4.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Notes - ----- - The position is not zero based, but 1 based index. Returns 0 if the given - value could not be found in the array. - Parameters ---------- - col : :class:`~pyspark.sql.Column` or str + col : :class:`~pyspark.sql.Column` or column name target column to work on. - A column that evaluates to an array. - value : Any - value or a :class:`~pyspark.sql.Column` expression to look for. - A column of the same type as the array elements. - - .. versionchanged:: 4.0.0 - `value` now also accepts a Column type. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` - position of the value in the given array if found and 0 otherwise. - Returns a column that evaluates to a long. + length of the value. + Returns a column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.array_contains` + :meth:`pyspark.sql.functions.char_length` + :meth:`pyspark.sql.functions.character_length` Examples -------- - Example 1: Finding the position of a string in an array of strings - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) - >>> df.select(sf.array_position(df.data, "a")).show() - +-----------------------+ - |array_position(data, a)| - +-----------------------+ - | 3| - +-----------------------+ - - Example 2: Finding the position of a string in an empty array - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_position(df.data, "a")).show() - +-----------------------+ - |array_position(data, a)| - +-----------------------+ - | 0| - +-----------------------+ - - Example 3: Finding the position of an integer in an array of integers + >>> spark.createDataFrame([('ABC ',)], ['a']).select('*', sf.length('a')).show() + +----+---------+ + | a|length(a)| + +----+---------+ + |ABC | 4| + +----+---------+ + """ + return _invoke_function_over_columns("length", col) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_position(df.data, 2)).show() - +-----------------------+ - |array_position(data, 2)| - +-----------------------+ - | 2| - +-----------------------+ - Example 4: Finding the position of a non-existing value in an array +@_try_remote_functions +def octet_length(col: "ColumnOrName") -> Column: + """ + Calculates the byte length for the specified string column. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) - >>> df.select(sf.array_position(df.data, "d")).show() - +-----------------------+ - |array_position(data, d)| - +-----------------------+ - | 0| - +-----------------------+ + .. versionadded:: 3.3.0 - Example 5: Finding the position of a value in an array with nulls + .. versionchanged:: 3.4.0 + Supports Spark Connect. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([None, "b", "a"],)], ['data']) - >>> df.select(sf.array_position(df.data, "a")).show() - +-----------------------+ - |array_position(data, a)| - +-----------------------+ - | 3| - +-----------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + Source column or strings. + A column that evaluates to a string or binary. - Example 6: Finding the position of a column's value in an array of integers + Returns + ------- + :class:`~pyspark.sql.Column` + Byte length of the col + Returns a column that evaluates to an integer. + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([10, 20, 30], 20)], ['data', 'col']) - >>> df.select(sf.array_position(df.data, df.col)).show() - +-------------------------+ - |array_position(data, col)| - +-------------------------+ - | 2| - +-------------------------+ - + >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) + >>> df.select('*', sf.octet_length('cat')).show() + +---+-----------------+ + |cat|octet_length(cat)| + +---+-----------------+ + |cat| 3| + | 🐈| 4| + +---+-----------------+ """ - return _invoke_function_over_columns("array_position", col, lit(value)) + return _invoke_function_over_columns("octet_length", col) @_try_remote_functions -def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column: +def bit_length(col: "ColumnOrName") -> Column: """ - Array function: Returns the element of an array at the given (0-based) index. - If the index points outside of the array boundaries, then this function - returns NULL. + Calculates the bit length for the specified string column. - .. versionadded:: 3.4.0 + .. versionadded:: 3.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of the column containing the array. - A column that evaluates to an array. - index : :class:`~pyspark.sql.Column` or str or int - Index to check for in the array. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + Source column or strings. + A column that evaluates to a string or binary. Returns ------- :class:`~pyspark.sql.Column` - Value at the given position. - Returns a column of the element type of the input array. - - Notes - ----- - The position is not 1-based, but 0-based index. - Supports Spark Connect. - - See Also - -------- - :meth:`pyspark.sql.functions.element_at` - :meth:`pyspark.sql.functions.try_element_at` + Bit length of the col + Returns a column that evaluates to an integer. Examples -------- - Example 1: Getting an element at a fixed position - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.get(df.data, 1)).show() - +------------+ - |get(data, 1)| - +------------+ - | b| - +------------+ + >>> df = spark.createDataFrame([('cat',), ( '\U0001f408',)], ['cat']) + >>> df.select('*', sf.bit_length('cat')).show() + +---+---------------+ + |cat|bit_length(cat)| + +---+---------------+ + |cat| 24| + | 🐈| 32| + +---+---------------+ + """ + return _invoke_function_over_columns("bit_length", col) - Example 2: Getting an element at a position outside the array boundaries - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) - >>> df.select(sf.get(df.data, 3)).show() - +------------+ - |get(data, 3)| - +------------+ - | NULL| - +------------+ +@_try_remote_functions +def translate(srcCol: "ColumnOrName", matching: str, replace: str) -> Column: + """A function translate any character in the `srcCol` by a character in `matching`. + The characters in `replace` is corresponding to the characters in `matching`. + Translation will happen whenever any character in the string is matching with the character + in the `matching`. - Example 3: Getting an element at a position specified by another column + .. versionadded:: 1.5.0 - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) - >>> df.select(sf.get(df.data, df.index)).show() - +----------------+ - |get(data, index)| - +----------------+ - | c| - +----------------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. + Parameters + ---------- + srcCol : :class:`~pyspark.sql.Column` or column name + Source column or strings. + A column that evaluates to a string. + matching : str + matching characters. + A column that evaluates to a string. + replace : str + characters for replacement. If this is shorter than `matching` string then + those chars that don't have replacement will be dropped. + A column that evaluates to a string. - Example 4: Getting an element at a position calculated from another column + Returns + ------- + :class:`~pyspark.sql.Column` + replaced value. + Returns a column that evaluates to a string. + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) - >>> df.select(sf.get(df.data, df.index - 1)).show() - +----------------------+ - |get(data, (index - 1))| - +----------------------+ - | b| - +----------------------+ + >>> df = spark.createDataFrame([('translate',)], ['a']) + >>> df.select('*', sf.translate('a', "rnlt", "123")).show() + +---------+-----------------------+ + | a|translate(a, rnlt, 123)| + +---------+-----------------------+ + |translate| 1a2s3ae| + +---------+-----------------------+ + """ + from pyspark.sql.classic.column import _to_java_column - Example 5: Getting an element at a negative position - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(["a", "b", "c"], )], ['data']) - >>> df.select(sf.get(df.data, -1)).show() - +-------------+ - |get(data, -1)| - +-------------+ - | NULL| - +-------------+ - """ - index = _enum_to_value(index) - index = lit(index) if isinstance(index, int) else index - - return _invoke_function_over_columns("get", col, index) + return _invoke_function( + "translate", _to_java_column(srcCol), _enum_to_value(matching), _enum_to_value(replace) + ) @_try_remote_functions -def array_prepend(col: "ColumnOrName", value: Any) -> Column: +def to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: """ - Array function: Returns an array containing the given element as - the first element and the rest of the elements from the original array. + Converts the input `col` to a binary value based on the supplied `format`. + The `format` can be a case-insensitive string literal of "hex", "utf-8", "utf8", + or "base64". By default, the binary format for conversion is "hex" if + `format` is omitted. The function returns NULL if at least one of the + input parameters is NULL. .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array. - A column that evaluates to an array. - value : - a literal value, or a :class:`~pyspark.sql.Column` expression. - A column of the same type as the array elements. - - Returns - ------- - :class:`~pyspark.sql.Column` - an array with the given value prepended. - Returns a column that evaluates to an array. + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert binary values. + A column that evaluates to a string. Must be a constant. See Also -------- - :meth:`pyspark.sql.functions.array_append` - :meth:`pyspark.sql.functions.array_insert` + :meth:`pyspark.sql.functions.try_to_binary` Examples -------- - Example 1: Prepending a column value to an array column - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) - >>> df.select(sf.array_prepend(df.c1, df.c2)).show() - +---------------------+ - |array_prepend(c1, c2)| - +---------------------+ - | [c, b, a, c]| - +---------------------+ + Example 1: Convert string to a binary with encoding specified - Example 2: Prepending a numeric value to an array column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("abc",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() + [Row(r=b'abc')] - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_prepend(df.data, 4)).show() - +----------------------+ - |array_prepend(data, 4)| - +----------------------+ - | [4, 1, 2, 3]| - +----------------------+ + Example 2: Convert string to a timestamp without encoding specified - Example 3: Prepending a null value to an array column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("414243",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() + [Row(r=b'ABC')] + """ + if format is not None: + return _invoke_function_over_columns("to_binary", col, format) + else: + return _invoke_function_over_columns("to_binary", col) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_prepend(df.data, None)).show() - +-------------------------+ - |array_prepend(data, NULL)| - +-------------------------+ - | [NULL, 1, 2, 3]| - +-------------------------+ - Example 4: Prepending a value to a NULL array column +@_try_remote_functions +def to_char(col: "ColumnOrName", format: "ColumnOrName") -> Column: + """ + Convert `col` to a string based on the `format`. + Throws an exception if the conversion fails. The format can consist of the following + characters, case insensitive: + '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the + format string matches a sequence of digits in the input value, generating a result + string of the same length as the corresponding sequence in the format string. + The result string is left-padded with zeros if the 0/9 sequence comprises more digits + than the matching part of the decimal value, starts with 0, and is before the decimal + point. Otherwise, it is padded with spaces. + '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). + ',' or 'G': Specifies the position of the grouping (thousands) separator (,). + There must be a 0 or 9 to the left and right of each grouping separator. + '$': Specifies the location of the $ currency sign. This character may only be specified once. + 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at + the beginning or end of the format string). Note that 'S' prints '+' for positive + values but 'MI' prints a space. + 'PR': Only allowed at the end of the format string; specifies that the result string + will be wrapped by angle brackets if the input value is negative. + If `col` is a datetime, `format` shall be a valid datetime pattern, see + Patterns. + If `col` is a binary, it is converted to a string in one of the formats: + 'base64': a base 64 string. + 'hex': a string in the hexadecimal format. + 'utf-8': the input binary is decoded to UTF-8 string. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([(None,)], schema=schema) - >>> df.select(sf.array_prepend(df.data, 4)).show() - +----------------------+ - |array_prepend(data, 4)| - +----------------------+ - | NULL| - +----------------------+ + .. versionadded:: 3.5.0 - Example 5: Prepending a value to an empty array + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert char values. + A column that evaluates to a string. Must be a constant when ``col`` is numeric + or binary. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_prepend(df.data, 1)).show() - +----------------------+ - |array_prepend(data, 1)| - +----------------------+ - | [1]| - +----------------------+ + Examples + -------- + >>> df = spark.createDataFrame([(78.12,)], ["e"]) + >>> df.select(to_char(df.e, lit("$99.99")).alias('r')).collect() + [Row(r='$78.12')] """ - return _invoke_function_over_columns("array_prepend", col, lit(value)) + return _invoke_function_over_columns("to_char", col, format) @_try_remote_functions -def array_remove(col: "ColumnOrName", element: Any) -> Column: +def to_varchar(col: "ColumnOrName", format: "ColumnOrName") -> Column: """ - Array function: Remove all elements that equal to element from the given array. - - .. versionadded:: 2.4.0 + Convert `col` to a string based on the `format`. + Throws an exception if the conversion fails. The format can consist of the following + characters, case insensitive: + '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the + format string matches a sequence of digits in the input value, generating a result + string of the same length as the corresponding sequence in the format string. + The result string is left-padded with zeros if the 0/9 sequence comprises more digits + than the matching part of the decimal value, starts with 0, and is before the decimal + point. Otherwise, it is padded with spaces. + '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). + ',' or 'G': Specifies the position of the grouping (thousands) separator (,). + There must be a 0 or 9 to the left and right of each grouping separator. + '$': Specifies the location of the $ currency sign. This character may only be specified once. + 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at + the beginning or end of the format string). Note that 'S' prints '+' for positive + values but 'MI' prints a space. + 'PR': Only allowed at the end of the format string; specifies that the result string + will be wrapped by angle brackets if the input value is negative. + If `col` is a datetime, `format` shall be a valid datetime pattern, see + Patterns. + If `col` is a binary, it is converted to a string in one of the formats: + 'base64': a base 64 string. + 'hex': a string in the hexadecimal format. + 'utf-8': the input binary is decoded to UTF-8 string. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or str - name of column containing array. - A column that evaluates to an array. - element : - element or a :class:`~pyspark.sql.Column` expression to be removed from the array. - A column of the same type as the array elements. - - .. versionchanged:: 4.0.0 - `element` now also accepts a Column type. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that is an array excluding the given value from the input column. - Returns a column that evaluates to an array. - - See Also - -------- - :meth:`pyspark.sql.functions.array_compact` + The value to convert to a string. + A column that evaluates to a numeric, date, timestamp, time, or binary. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert char values. + A column that evaluates to a string. Must be a constant when ``col`` is numeric + or binary. Examples -------- - Example 1: Removing a specific value from a simple array + >>> df = spark.createDataFrame([(78.12,)], ["e"]) + >>> df.select(to_varchar(df.e, lit("$99.99")).alias('r')).collect() + [Row(r='$78.12')] + """ + return _invoke_function_over_columns("to_varchar", col, format) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],)], ['data']) - >>> df.select(sf.array_remove(df.data, 1)).show() - +---------------------+ - |array_remove(data, 1)| - +---------------------+ - | [2, 3]| - +---------------------+ - Example 2: Removing a specific value from multiple arrays +@_try_remote_functions +def to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: + """ + Convert string 'col' to a number based on the string format 'format'. + Throws an exception if the conversion fails. The format can consist of the following + characters, case insensitive: + '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the + format string matches a sequence of digits in the input string. If the 0/9 + sequence starts with 0 and is before the decimal point, it can only match a digit + sequence of the same size. Otherwise, if the sequence starts with 9 or is after + the decimal point, it can match a digit sequence that has the same or smaller size. + '.' or 'D': Specifies the position of the decimal point (optional, only allowed once). + ',' or 'G': Specifies the position of the grouping (thousands) separator (,). + There must be a 0 or 9 to the left and right of each grouping separator. + 'col' must match the grouping separator relevant for the size of the number. + '$': Specifies the location of the $ currency sign. This character may only be + specified once. + 'S' or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed + once at the beginning or end of the format string). Note that 'S' allows '-' + but 'MI' does not. + 'PR': Only allowed at the end of the format string; specifies that 'col' indicates a + negative number with wrapping angled brackets. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],), ([4, 5, 5, 4],)], ['data']) - >>> df.select(sf.array_remove(df.data, 5)).show() - +---------------------+ - |array_remove(data, 5)| - +---------------------+ - | [1, 2, 3, 1, 1]| - | [4, 4]| - +---------------------+ + .. versionadded:: 3.5.0 - Example 3: Removing a value that does not exist in the array + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert number values. + A column that evaluates to a string. Must be a constant. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_remove(df.data, 4)).show() - +---------------------+ - |array_remove(data, 4)| - +---------------------+ - | [1, 2, 3]| - +---------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.try_to_number` - Example 4: Removing a value from an array with all identical values + Examples + -------- + >>> df = spark.createDataFrame([("$78.12",)], ["e"]) + >>> df.select(to_number(df.e, lit("$99.99")).alias('r')).collect() + [Row(r=Decimal('78.12'))] + """ + return _invoke_function_over_columns("to_number", col, format) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) - >>> df.select(sf.array_remove(df.data, 1)).show() - +---------------------+ - |array_remove(data, 1)| - +---------------------+ - | []| - +---------------------+ - Example 5: Removing a value from an empty array +@_try_remote_functions +def replace( + src: "ColumnOrName", search: "ColumnOrName", replace: Optional["ColumnOrName"] = None +) -> Column: + """ + Replaces all occurrences of `search` with `replace`. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema) - >>> df.select(sf.array_remove(df.data, 1)).show() - +---------------------+ - |array_remove(data, 1)| - +---------------------+ - | []| - +---------------------+ + .. versionadded:: 3.5.0 - Example 6: Removing a column's value from a simple array + Parameters + ---------- + src : :class:`~pyspark.sql.Column` or str + A column of string to be replaced. + A column that evaluates to a string. + search : :class:`~pyspark.sql.Column` or str + A column of string, If `search` is not found in `str`, `str` is returned unchanged. + A column that evaluates to a string. + replace : :class:`~pyspark.sql.Column` or str, optional + A column of string, If `replace` is not specified or is an empty string, + nothing replaces the string that is removed from `str`. + A column that evaluates to a string. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 1, 1], 1)], ['data', 'col']) - >>> df.select(sf.array_remove(df.data, df.col)).show() - +-----------------------+ - |array_remove(data, col)| - +-----------------------+ - | [2, 3]| - +-----------------------+ + Examples + -------- + >>> df = spark.createDataFrame([("ABCabc", "abc", "DEF",)], ["a", "b", "c"]) + >>> df.select(replace(df.a, df.b, df.c).alias('r')).collect() + [Row(r='ABCDEF')] + + >>> df.select(replace(df.a, df.b).alias('r')).collect() + [Row(r='ABC')] """ - return _invoke_function_over_columns("array_remove", col, lit(element)) + if replace is not None: + return _invoke_function_over_columns("replace", src, search, replace) + else: + return _invoke_function_over_columns("replace", src, search) @_try_remote_functions -def array_distinct(col: "ColumnOrName") -> Column: +def split_part(src: "ColumnOrName", delimiter: "ColumnOrName", partNum: "ColumnOrName") -> Column: """ - Array function: removes duplicate values from the array. - - .. versionadded:: 2.4.0 + Splits `str` by delimiter and return requested part of the split (1-based). + If any input is null, returns null. if `partNum` is out of range of split parts, + returns empty string. If `partNum` is 0, throws an error. If `partNum` is negative, + the parts are counted backward from the end of the string. + If the `delimiter` is an empty string, the `str` is not split. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that is an array of unique values from the input column. - Returns a column that evaluates to an array. + src : :class:`~pyspark.sql.Column` or column name + A column of string to be split. + A column that evaluates to a string. + delimiter : :class:`~pyspark.sql.Column` or column name + A column of string, the delimiter used for split. + A column that evaluates to a string. + partNum : :class:`~pyspark.sql.Column` or column name + The requested part of the split (1-based). + A column that evaluates to an integer. See Also -------- - :meth:`pyspark.sql.functions.array_except` - :meth:`pyspark.sql.functions.array_intersect` - :meth:`pyspark.sql.functions.array_union` + :meth:`pyspark.sql.functions.sentences` + :meth:`pyspark.sql.functions.split` Examples -------- - Example 1: Removing duplicate values from a simple array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 2],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1, 2, 3]| - +--------------------+ - - Example 2: Removing duplicate values from multiple arrays - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1, 2, 3]| - | [4, 5]| - +--------------------+ - - Example 3: Removing duplicate values from an array with all identical values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1]| - +--------------------+ - - Example 4: Removing duplicate values from an array with no duplicate values - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | [1, 2, 3]| - +--------------------+ - - Example 5: Removing duplicate values from an empty array + >>> df = spark.createDataFrame([("11.12.13", ".", 3,)], ["a", "b", "c"]) + >>> df.select("*", sf.split_part("a", "b", "c")).show() + +--------+---+---+-------------------+ + | a| b| c|split_part(a, b, c)| + +--------+---+---+-------------------+ + |11.12.13| .| 3| 13| + +--------+---+---+-------------------+ - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema) - >>> df.select(sf.array_distinct(df.data)).show() - +--------------------+ - |array_distinct(data)| - +--------------------+ - | []| - +--------------------+ + >>> df.select("*", sf.split_part(df.a, df.b, sf.lit(-2))).show() + +--------+---+---+--------------------+ + | a| b| c|split_part(a, b, -2)| + +--------+---+---+--------------------+ + |11.12.13| .| 3| 12| + +--------+---+---+--------------------+ """ - return _invoke_function_over_columns("array_distinct", col) + return _invoke_function_over_columns("split_part", src, delimiter, partNum) @_try_remote_functions -def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: Any) -> Column: +def substr( + str: "ColumnOrName", pos: "ColumnOrName", len: Optional["ColumnOrName"] = None +) -> Column: """ - Array function: Inserts an item into a given array at a specified array index. - Array indices start at 1, or start from the end if index is negative. - Index above array size appends the array, or prepends the array if index is negative, - with 'null' elements. + Returns the substring of `str` that starts at `pos` and is of length `len`, + or the slice of byte array that starts at `pos` and is of length `len`. - .. versionadded:: 3.4.0 + .. versionadded:: 3.5.0 Parameters ---------- - arr : :class:`~pyspark.sql.Column` or str - name of column containing an array. - A column that evaluates to an array. - pos : :class:`~pyspark.sql.Column` or str or int - name of integral type column indicating position of insertion - (starting at index 1, negative position is a start from the back of the array). + str : :class:`~pyspark.sql.Column` or column name + A column of string. + A column that evaluates to a string or binary. + pos : :class:`~pyspark.sql.Column` or column name + The starting position of the substring. + A column that evaluates to an integer. + len : :class:`~pyspark.sql.Column` or column name, optional + The length of the substring. A column that evaluates to an integer. - value : - a literal value, or a :class:`~pyspark.sql.Column` expression. - A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` - an array of values, including the new specified value - Returns a column that evaluates to an array. - - Notes - ----- - Supports Spark Connect. + substring of given value. + Returns a column of the same type as the input. See Also -------- - :meth:`pyspark.sql.functions.array_append` - :meth:`pyspark.sql.functions.array_prepend` + :meth:`pyspark.sql.functions.instr` + :meth:`pyspark.sql.functions.substring` + :meth:`pyspark.sql.functions.substring_index` + :meth:`pyspark.sql.Column.substr` + :meth:`pyspark.sql.functions.locate` Examples -------- - Example 1: Inserting a value at a specific position - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, 2, 'd')).show() - +------------------------+ - |array_insert(data, 2, d)| - +------------------------+ - | [a, d, b, c]| - +------------------------+ + >>> df = spark.createDataFrame([("Spark SQL", 5, 1,)], ["a", "b", "c"]) + >>> df.select("*", sf.substr("a", "b", "c")).show() + +---------+---+---+---------------+ + | a| b| c|substr(a, b, c)| + +---------+---+---+---------------+ + |Spark SQL| 5| 1| k| + +---------+---+---+---------------+ - Example 2: Inserting a value at a negative position + >>> df.select("*", sf.substr(df.a, df.b)).show() + +---------+---+---+------------------------+ + | a| b| c|substr(a, b, 2147483647)| + +---------+---+---+------------------------+ + |Spark SQL| 5| 1| k SQL| + +---------+---+---+------------------------+ + """ + if len is not None: + return _invoke_function_over_columns("substr", str, pos, len) + else: + return _invoke_function_over_columns("substr", str, pos) + + +@_try_remote_functions +def try_parse_url( + url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None +) -> Column: + """ + This is a special version of `parse_url` that performs the same operation, but returns a + NULL value instead of raising an error if the parsing cannot be performed. + + .. versionadded:: 4.0.0 + + Parameters + ---------- + url : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL. + A column that evaluates to a string. + partToExtract : :class:`~pyspark.sql.Column` or str + A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. + key : :class:`~pyspark.sql.Column` or str, optional + A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. + + Examples + -------- + Example 1: Extracting the query part from a URL >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, -2, 'd')).show() - +-------------------------+ - |array_insert(data, -2, d)| - +-------------------------+ - | [a, b, d, c]| - +-------------------------+ + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "QUERY")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() + +------------------------+ + |try_parse_url(url, part)| + +------------------------+ + | query=1| + +------------------------+ - Example 3: Inserting a value at a position greater than the array size + Example 2: Extracting the value of a specific query parameter from a URL >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, 5, 'e')).show() + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], + ... ["url", "part", "key"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() + +-----------------------------+ + |try_parse_url(url, part, key)| + +-----------------------------+ + | 1| + +-----------------------------+ + + Example 3: Extracting the protocol part from a URL + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() +------------------------+ - |array_insert(data, 5, e)| + |try_parse_url(url, part)| +------------------------+ - | [a, b, c, NULL, e]| + | https| +------------------------+ - Example 4: Inserting a NULL value + Example 4: Extracting the host part from a URL >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) - >>> df.select(sf.array_insert(df.data, 2, sf.lit(None))).show() - +---------------------------+ - |array_insert(data, 2, NULL)| - +---------------------------+ - | [a, NULL, b, c]| - +---------------------------+ + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "HOST")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() + +------------------------+ + |try_parse_url(url, part)| + +------------------------+ + | spark.apache.org| + +------------------------+ - Example 5: Inserting a value into a NULL array + Example 5: Extracting the path part from a URL >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) - >>> df = spark.createDataFrame([(None,)], schema=schema) - >>> df.select(sf.array_insert(df.data, 1, 5)).show() + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PATH")], + ... ["url", "part"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part)).show() +------------------------+ - |array_insert(data, 1, 5)| + |try_parse_url(url, part)| +------------------------+ - | NULL| + | /path| +------------------------+ - """ - pos = _enum_to_value(pos) - pos = lit(pos) if isinstance(pos, int) else pos - return _invoke_function_over_columns("array_insert", arr, pos, lit(value)) + Example 6: Invalid URL + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("inva lid://spark.apache.org/path?query=1", "QUERY", "query")], + ... ["url", "part", "key"] + ... ) + >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() + +-----------------------------+ + |try_parse_url(url, part, key)| + +-----------------------------+ + | NULL| + +-----------------------------+ + """ + if key is not None: + return _invoke_function_over_columns("try_parse_url", url, partToExtract, key) + else: + return _invoke_function_over_columns("try_parse_url", url, partToExtract) @_try_remote_functions -def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def parse_url( + url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None +) -> Column: """ - Array function: returns a new array containing the intersection of elements in col1 and col2, - without duplicates. - - .. versionadded:: 2.4.0 + URL function: Extracts a specified part from a URL. If a key is provided, + it returns the associated query parameter value. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - Name of column containing the first array. - A column that evaluates to an array. - col2 : :class:`~pyspark.sql.Column` or str - Name of column containing the second array. - A column that evaluates to an array. + url : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL. + A column that evaluates to a string. + partToExtract : :class:`~pyspark.sql.Column` or str + A column of strings, each representing the part to extract from the URL. + A column that evaluates to a string. + key : :class:`~pyspark.sql.Column` or str, optional + A column of strings, each representing the key of a query parameter in the URL. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new array containing the intersection of elements in col1 and col2. - Returns a column that evaluates to an array. - - Notes - ----- - This function does not preserve the order of the elements in the input arrays. - - See Also - -------- - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.array_except` - :meth:`pyspark.sql.functions.array_union` + A new column of strings, each representing the value of the extracted part from the URL. + Returns a column that evaluates to a string. Examples -------- - Example 1: Basic usage + Example 1: Extracting the query part from a URL - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) - >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() - +-----------------------------------------+ - |sort_array(array_intersect(c1, c2), true)| - +-----------------------------------------+ - | [a, c]| - +-----------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "QUERY")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | query=1| + +--------------------+ - Example 2: Intersection with no common elements + Example 2: Extracting the value of a specific query parameter from a URL - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) - >>> df.select(sf.array_intersect(df.c1, df.c2)).show() - +-----------------------+ - |array_intersect(c1, c2)| - +-----------------------+ - | []| - +-----------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], + ... ["url", "part", "key"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part, df.key)).show() + +-------------------------+ + |parse_url(url, part, key)| + +-------------------------+ + | 1| + +-------------------------+ - Example 3: Intersection with all common elements + Example 3: Extracting the protocol part from a URL - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) - >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() - +-----------------------------------------+ - |sort_array(array_intersect(c1, c2), true)| - +-----------------------------------------+ - | [a, b, c]| - +-----------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | https| + +--------------------+ - Example 4: Intersection with null values + Example 4: Extracting the host part from a URL - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) - >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() - +-----------------------------------------+ - |sort_array(array_intersect(c1, c2), true)| - +-----------------------------------------+ - | [NULL, a]| - +-----------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "HOST")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | spark.apache.org| + +--------------------+ - Example 5: Intersection with empty arrays + Example 5: Extracting the path part from a URL - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> data = [Row(c1=[], c2=["a", "b", "c"])] - >>> schema = StructType([ - ... StructField("c1", ArrayType(StringType()), True), - ... StructField("c2", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.array_intersect(df.c1, df.c2)).show() - +-----------------------+ - |array_intersect(c1, c2)| - +-----------------------+ - | []| - +-----------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [("https://spark.apache.org/path?query=1", "PATH")], + ... ["url", "part"] + ... ) + >>> df.select(sf.parse_url(df.url, df.part)).show() + +--------------------+ + |parse_url(url, part)| + +--------------------+ + | /path| + +--------------------+ """ - return _invoke_function_over_columns("array_intersect", col1, col2) + if key is not None: + return _invoke_function_over_columns("parse_url", url, partToExtract, key) + else: + return _invoke_function_over_columns("parse_url", url, partToExtract) @_try_remote_functions -def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def printf(format: "ColumnOrName", *cols: "ColumnOrName") -> Column: """ - Array function: returns a new array containing the union of elements in col1 and col2, - without duplicates. + Formats the arguments in printf-style and returns the result as a string column. - .. versionadded:: 2.4.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Parameters + ---------- + format : :class:`~pyspark.sql.Column` or str + string that can contain embedded format tags and used as result column's value. + A column that evaluates to a string. + cols : :class:`~pyspark.sql.Column` or str + column names or :class:`~pyspark.sql.Column`\\s to be used in formatting + Each a column of any type. + + See Also + -------- + :meth:`pyspark.sql.functions.format_string` + + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("aa%d%s", 123, "cc",)], ["a", "b", "c"] + ... ).select(sf.printf("a", "b", "c")).show() + +---------------+ + |printf(a, b, c)| + +---------------+ + | aa123cc| + +---------------+ + """ + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function("printf", _to_java_column(format), _to_seq(sc, cols, _to_java_column)) + + +@_try_remote_functions +def url_decode(str: "ColumnOrName") -> Column: + """ + URL function: Decodes a URL-encoded string in 'application/x-www-form-urlencoded' + format to its original format. + + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - Name of column containing the first array. - A column that evaluates to an array. - col2 : :class:`~pyspark.sql.Column` or str - Name of column containing the second array. - A column that evaluates to an array. + str : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new array containing the union of elements in col1 and col2. - Returns a column that evaluates to an array. - - Notes - ----- - This function does not preserve the order of the elements in the input arrays. - - See Also - -------- - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.array_except` - :meth:`pyspark.sql.functions.array_intersect` + A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- - Example 1: Basic usage + Example 1: Decoding a URL-encoded string - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c, d, f]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show(truncate=False) + +------------------------+ + |url_decode(url) | + +------------------------+ + |https://spark.apache.org| + +------------------------+ - Example 2: Union with no common elements + Example 2: Decoding a URL-encoded string with spaces - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c, d, e, f]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Hello%20World%21",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | Hello World!| + +---------------+ - Example 3: Union with all common elements + Example 3: Decoding a URL-encoded string with special characters - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("A%2BB%3D%3D",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | A+B==| + +---------------+ - Example 4: Union with null values + Example 4: Decoding a URL-encoded string with non-ASCII characters - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [NULL, a, b, c]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("%E4%BD%A0%E5%A5%BD",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | 你好| + +---------------+ - Example 5: Union with empty arrays + Example 5: Decoding a URL-encoded string with hexadecimal values - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> data = [Row(c1=[], c2=["a", "b", "c"])] - >>> schema = StructType([ - ... StructField("c1", ArrayType(StringType()), True), - ... StructField("c2", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() - +-------------------------------------+ - |sort_array(array_union(c1, c2), true)| - +-------------------------------------+ - | [a, b, c]| - +-------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("%7E%21%40%23%24%25%5E%26%2A%28%29%5F%2B",)], ["url"]) + >>> df.select(sf.url_decode(df.url)).show() + +---------------+ + |url_decode(url)| + +---------------+ + | ~!@#$%^&*()_+| + +---------------+ """ - return _invoke_function_over_columns("array_union", col1, col2) + return _invoke_function_over_columns("url_decode", str) @_try_remote_functions -def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def try_url_decode(str: "ColumnOrName") -> Column: """ - Array function: returns a new array containing the elements present in col1 but not in col2, - without duplicates. - - .. versionadded:: 2.4.0 + This is a special version of `url_decode` that performs the same operation, but returns a + NULL value instead of raising an error if the decoding cannot be performed. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or str - Name of column containing the first array. - A column that evaluates to an array. - col2 : :class:`~pyspark.sql.Column` or str - Name of column containing the second array. - A column that evaluates to an array. + str : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a URL-encoded string. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new array containing the elements present in col1 but not in col2. - Returns a column that evaluates to an array. - - Notes - ----- - This function does not preserve the order of the elements in the input arrays. - - See Also - -------- - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.array_intersect` - :meth:`pyspark.sql.functions.array_union` + A new column of strings, each representing the decoded string. + Returns a column that evaluates to a string. Examples -------- - Example 1: Basic usage - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) - >>> df.select(sf.array_except(df.c1, df.c2)).show() - +--------------------+ - |array_except(c1, c2)| - +--------------------+ - | [b]| - +--------------------+ - - Example 2: Except with no common elements - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) - >>> df.select(sf.sort_array(sf.array_except(df.c1, df.c2))).show() - +--------------------------------------+ - |sort_array(array_except(c1, c2), true)| - +--------------------------------------+ - | [a, b, c]| - +--------------------------------------+ - - Example 3: Except with all common elements - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) - >>> df.select(sf.array_except(df.c1, df.c2)).show() - +--------------------+ - |array_except(c1, c2)| - +--------------------+ - | []| - +--------------------+ - - Example 4: Except with null values + Example 1: Decoding a URL-encoded string - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) - >>> df.select(sf.array_except(df.c1, df.c2)).show() - +--------------------+ - |array_except(c1, c2)| - +--------------------+ - | [b]| - +--------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) + >>> df.select(sf.try_url_decode(df.url)).show(truncate=False) + +------------------------+ + |try_url_decode(url) | + +------------------------+ + |https://spark.apache.org| + +------------------------+ - Example 5: Except with empty arrays + Example 2: Return NULL if the decoding cannot be performed. - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> data = [Row(c1=[], c2=["a", "b", "c"])] - >>> schema = StructType([ - ... StructField("c1", ArrayType(StringType()), True), - ... StructField("c2", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.array_except(df.c1, df.c2)).show() - +--------------------+ - |array_except(c1, c2)| - +--------------------+ - | []| - +--------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("https%3A%2F%2spark.apache.org",)], ["url"]) + >>> df.select(sf.try_url_decode(df.url)).show() + +-------------------+ + |try_url_decode(url)| + +-------------------+ + | NULL| + +-------------------+ """ - return _invoke_function_over_columns("array_except", col1, col2) + return _invoke_function_over_columns("try_url_decode", str) @_try_remote_functions -def array_compact(col: "ColumnOrName") -> Column: +def url_encode(str: "ColumnOrName") -> Column: """ - Array function: removes null values from the array. + URL function: Encodes a string into a URL-encoded string in + 'application/x-www-form-urlencoded' format. - .. versionadded:: 3.4.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column or expression. - A column that evaluates to an array. + str : :class:`~pyspark.sql.Column` or str + A column of strings, each representing a string to be URL-encoded. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new column that is an array excluding the null values from the input column. - Returns a column that evaluates to an array. - - Notes - ----- - Supports Spark Connect. - - See Also - -------- - :meth:`pyspark.sql.functions.array_remove` + A new column of strings, each representing the URL-encoded string. + Returns a column that evaluates to a string. Examples -------- - Example 1: Removing null values from a simple array + Example 1: Encoding a simple URL >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, None, 2, 3],)], ['data']) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | [1, 2, 3]| - +-------------------+ + >>> df = spark.createDataFrame([("https://spark.apache.org",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show(truncate=False) + +------------------------------+ + |url_encode(url) | + +------------------------------+ + |https%3A%2F%2Fspark.apache.org| + +------------------------------+ - Example 2: Removing null values from multiple arrays + Example 2: Encoding a URL with spaces >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, None, 2, 3],), ([4, 5, None, 4],)], ['data']) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | [1, 2, 3]| - | [4, 5, 4]| - +-------------------+ + >>> df = spark.createDataFrame([("Hello World!",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show() + +---------------+ + |url_encode(url)| + +---------------+ + | Hello+World%21| + +---------------+ - Example 3: Removing null values from an array with all null values + Example 3: Encoding a URL with special characters >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([ - ... StructField("data", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame([([None, None, None],)], schema) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | []| - +-------------------+ + >>> df = spark.createDataFrame([("A+B==",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show() + +---------------+ + |url_encode(url)| + +---------------+ + | A%2BB%3D%3D| + +---------------+ - Example 4: Removing null values from an array with no null values + Example 4: Encoding a URL with non-ASCII characters >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | [1, 2, 3]| - +-------------------+ + >>> df = spark.createDataFrame([("你好",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show() + +------------------+ + | url_encode(url)| + +------------------+ + |%E4%BD%A0%E5%A5%BD| + +------------------+ - Example 5: Removing null values from an empty array + Example 5: Encoding a URL with hexadecimal values >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([ - ... StructField("data", ArrayType(StringType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema) - >>> df.select(sf.array_compact(df.data)).show() - +-------------------+ - |array_compact(data)| - +-------------------+ - | []| - +-------------------+ + >>> df = spark.createDataFrame([("~!@#$%^&*()_+",)], ["url"]) + >>> df.select(sf.url_encode(df.url)).show(truncate=False) + +-----------------------------------+ + |url_encode(url) | + +-----------------------------------+ + |%7E%21%40%23%24%25%5E%26*%28%29_%2B| + +-----------------------------------+ """ - return _invoke_function_over_columns("array_compact", col) + return _invoke_function_over_columns("url_encode", str) @_try_remote_functions -def array_append(col: "ColumnOrName", value: Any) -> Column: +def position( + substr: "ColumnOrName", str: "ColumnOrName", start: Optional["ColumnOrName"] = None +) -> Column: """ - Array function: returns a new array column by appending `value` to the existing array `col`. + Returns the position of the first occurrence of `substr` in `str` after position `start`. + The given `start` and return value are 1-based. - .. versionadded:: 3.4.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column containing the array. - A column that evaluates to an array. - value : - A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. - A column of the same type as the array elements. + substr : :class:`~pyspark.sql.Column` or str + A column of string, substring. + A column that evaluates to a string. + str : :class:`~pyspark.sql.Column` or str + A column of string. + A column that evaluates to a string. + start : :class:`~pyspark.sql.Column` or str, optional + The start position. + A column that evaluates to an integer. - Returns - ------- - :class:`~pyspark.sql.Column` - A new array column with `value` appended to the original array. - Returns a column that evaluates to an array. + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] + ... ).select(sf.position("a", "b", "c")).show() + +-----------------+ + |position(a, b, c)| + +-----------------+ + | 7| + +-----------------+ - Notes - ----- - Supports Spark Connect. + >>> spark.createDataFrame( + ... [("bar", "foobarbar", 5,)], ["a", "b", "c"] + ... ).select(sf.position("a", "b")).show() + +-----------------+ + |position(a, b, 1)| + +-----------------+ + | 4| + +-----------------+ + """ + if start is not None: + return _invoke_function_over_columns("position", substr, str, start) + else: + return _invoke_function_over_columns("position", substr, str) - See Also - -------- - :meth:`pyspark.sql.functions.array_insert` - :meth:`pyspark.sql.functions.array_prepend` - Examples - -------- - Example 1: Appending a column value to an array column +@_try_remote_functions +def endswith(str: "ColumnOrName", suffix: "ColumnOrName") -> Column: + """ + Returns a boolean. The value is True if str ends with suffix. + Returns NULL if either input expression is NULL. Otherwise, returns False. + Both str or suffix must be of STRING or BINARY type. - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) - >>> df.select(sf.array_append(df.c1, df.c2)).show() - +--------------------+ - |array_append(c1, c2)| - +--------------------+ - | [b, a, c, c]| - +--------------------+ + .. versionadded:: 3.5.0 - Example 2: Appending a numeric value to an array column + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + The input value to test. + A column that evaluates to a string or binary. + suffix : :class:`~pyspark.sql.Column` or str + The suffix to test for. + A column that evaluates to a string or binary. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_append(df.data, 4)).show() - +---------------------+ - |array_append(data, 4)| - +---------------------+ - | [1, 2, 3, 4]| - +---------------------+ + Examples + -------- + >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) + >>> df.select(endswith(df.a, df.b).alias('r')).collect() + [Row(r=False)] - Example 3: Appending a null value to an array column + >>> df = spark.createDataFrame([("414243", "4243",)], ["e", "f"]) + >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) + >>> df.printSchema() + root + |-- e: binary (nullable = true) + |-- f: binary (nullable = true) + >>> df.select(endswith("e", "f"), endswith("f", "e")).show() + +--------------+--------------+ + |endswith(e, f)|endswith(f, e)| + +--------------+--------------+ + | true| false| + +--------------+--------------+ + """ + return _invoke_function_over_columns("endswith", str, suffix) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) - >>> df.select(sf.array_append(df.data, None)).show() - +------------------------+ - |array_append(data, NULL)| - +------------------------+ - | [1, 2, 3, NULL]| - +------------------------+ - Example 4: Appending a value to a NULL array column +@_try_remote_functions +def startswith(str: "ColumnOrName", prefix: "ColumnOrName") -> Column: + """ + Returns a boolean. The value is True if str starts with prefix. + Returns NULL if either input expression is NULL. Otherwise, returns False. + Both str or prefix must be of STRING or BINARY type. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([(None,)], schema=schema) - >>> df.select(sf.array_append(df.data, 4)).show() - +---------------------+ - |array_append(data, 4)| - +---------------------+ - | NULL| - +---------------------+ + .. versionadded:: 3.5.0 - Example 5: Appending a value to an empty array + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + The input value to test. + A column that evaluates to a string or binary. + prefix : :class:`~pyspark.sql.Column` or str + The prefix to test for. + A column that evaluates to a string or binary. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_append(df.data, 1)).show() - +---------------------+ - |array_append(data, 1)| - +---------------------+ - | [1]| - +---------------------+ + Examples + -------- + >>> df = spark.createDataFrame([("Spark SQL", "Spark",)], ["a", "b"]) + >>> df.select(startswith(df.a, df.b).alias('r')).collect() + [Row(r=True)] + + >>> df = spark.createDataFrame([("414243", "4142",)], ["e", "f"]) + >>> df = df.select(to_binary("e").alias("e"), to_binary("f").alias("f")) + >>> df.printSchema() + root + |-- e: binary (nullable = true) + |-- f: binary (nullable = true) + >>> df.select(startswith("e", "f"), startswith("f", "e")).show() + +----------------+----------------+ + |startswith(e, f)|startswith(f, e)| + +----------------+----------------+ + | true| false| + +----------------+----------------+ """ - return _invoke_function_over_columns("array_append", col, lit(value)) + return _invoke_function_over_columns("startswith", str, prefix) @_try_remote_functions -def array_min(col: "ColumnOrName") -> Column: +def char(col: "ColumnOrName") -> Column: """ - Array function: returns the minimum value of the array. - - .. versionadded:: 2.4.0 + Returns the ASCII character having the binary equivalent to `col`. If col is larger than 256 the + result is equivalent to char(col % 256) - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the array. - A column that evaluates to an array. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains the minimum value of each array. - Returns a column of the element type of the input array. - - See Also - -------- - :meth:`pyspark.sql.functions.array_max` - :meth:`pyspark.sql.functions.array_sort` - :meth:`pyspark.sql.functions.sort_array` + Input column or strings. + A column that evaluates to a long. Examples -------- - Example 1: Basic usage with integer array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | 1| - | -1| - +---------------+ - - Example 2: Usage with string array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | apple| - +---------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.char(sf.lit(65))).show() + +--------+ + |char(65)| + +--------+ + | A| + +--------+ + """ + return _invoke_function_over_columns("char", col) - Example 3: Usage with mixed type array - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | 1| - +---------------+ +@_try_remote_functions +def btrim(str: "ColumnOrName", trim: Optional["ColumnOrName"] = None) -> Column: + """ + Remove the leading and trailing `trim` characters from `str`. - Example 4: Usage with array of arrays + .. versionadded:: 3.5.0 - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | [2, 1]| - +---------------+ + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + trim : :class:`~pyspark.sql.Column` or str, optional + The trim string characters to trim, the default value is a single space. + A column that evaluates to a string. - Example 5: Usage with empty array + Examples + -------- + >>> df = spark.createDataFrame([("SSparkSQLS", "SL", )], ['a', 'b']) + >>> df.select(btrim(df.a, df.b).alias('r')).collect() + [Row(r='parkSQ')] - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_min(df.data)).show() - +---------------+ - |array_min(data)| - +---------------+ - | NULL| - +---------------+ + >>> df = spark.createDataFrame([(" SparkSQL ",)], ['a']) + >>> df.select(btrim(df.a).alias('r')).collect() + [Row(r='SparkSQL')] """ - return _invoke_function_over_columns("array_min", col) + if trim is not None: + return _invoke_function_over_columns("btrim", str, trim) + else: + return _invoke_function_over_columns("btrim", str) @_try_remote_functions -def array_max(col: "ColumnOrName") -> Column: +def char_length(str: "ColumnOrName") -> Column: """ - Array function: returns the maximum value of the array. - - .. versionadded:: 2.4.0 + Returns the character length of string data or number of bytes of binary data. + The length of string data includes the trailing spaces. + The length of binary data includes binary zeros. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the array. - A column that evaluates to an array. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains the maximum value of each array. - Returns a column of the element type of the input array. + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string or binary. See Also -------- - :meth:`pyspark.sql.functions.array_min` - :meth:`pyspark.sql.functions.array_sort` - :meth:`pyspark.sql.functions.sort_array` + :meth:`pyspark.sql.functions.character_length` + :meth:`pyspark.sql.functions.length` Examples -------- - Example 1: Basic usage with integer array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | 3| - | 10| - +---------------+ - - Example 2: Usage with string array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | cherry| - +---------------+ - - Example 3: Usage with mixed type array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | cherry| - +---------------+ - - Example 4: Usage with array of arrays - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | [3, 4]| - +---------------+ - - Example 5: Usage with empty array - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_max(df.data)).show() - +---------------+ - |array_max(data)| - +---------------+ - | NULL| - +---------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.char_length(sf.lit("SparkSQL"))).show() + +---------------------+ + |char_length(SparkSQL)| + +---------------------+ + | 8| + +---------------------+ """ - return _invoke_function_over_columns("array_max", col) + return _invoke_function_over_columns("char_length", str) @_try_remote_functions -def array_size(col: "ColumnOrName") -> Column: +def character_length(str: "ColumnOrName") -> Column: """ - Array function: returns the total number of elements in the array. - The function returns null for null input. + Returns the character length of string data or number of bytes of binary data. + The length of string data includes the trailing spaces. + The length of binary data includes binary zeros. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the array. - A column that evaluates to an array. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains the size of each array. - Returns a column that evaluates to an integer. + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string or binary. See Also -------- - :meth:`pyspark.sql.functions.cardinality` - :meth:`pyspark.sql.functions.size` + :meth:`pyspark.sql.functions.char_length` + :meth:`pyspark.sql.functions.length` Examples -------- - Example 1: Basic usage with integer array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([2, 1, 3],), (None,)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 3| - | NULL| - +----------------+ - - Example 2: Usage with string array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 3| - +----------------+ - - Example 3: Usage with mixed type array + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.character_length(sf.lit("SparkSQL"))).show() + +--------------------------+ + |character_length(SparkSQL)| + +--------------------------+ + | 8| + +--------------------------+ + """ + return _invoke_function_over_columns("character_length", str) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 3| - +----------------+ - Example 4: Usage with array of arrays +@_try_remote_functions +def chr(n: "ColumnOrName") -> Column: + """ + Returns the ASCII character having the binary equivalent to `n`. + If n is larger than 256 the result is equivalent to chr(n % 256). - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 2| - +----------------+ + .. versionadded:: 4.1.0 - Example 5: Usage with empty array + Parameters + ---------- + n : :class:`~pyspark.sql.Column` or column name + target column to compute on. + A column that evaluates to a long. - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType(IntegerType()), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.array_size(df.data)).show() - +----------------+ - |array_size(data)| - +----------------+ - | 0| - +----------------+ + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> spark.range(60, 70).select("*", sf.chr("id")).show() + +---+-------+ + | id|chr(id)| + +---+-------+ + | 60| <| + | 61| =| + | 62| >| + | 63| ?| + | 64| @| + | 65| A| + | 66| B| + | 67| C| + | 68| D| + | 69| E| + +---+-------+ """ - return _invoke_function_over_columns("array_size", col) + return _invoke_function_over_columns("chr", n) @_try_remote_functions -def sort_array(col: "ColumnOrName", asc: bool = True) -> Column: +def try_to_binary(col: "ColumnOrName", format: Optional["ColumnOrName"] = None) -> Column: """ - Array function: Sorts the input array in ascending or descending order according - to the natural ordering of the array elements. Null elements will be placed at the beginning - of the returned array in ascending order or at the end of the returned array in descending - order. - - .. versionadded:: 1.5.0 + This is a special version of `to_binary` that performs the same operation, but returns a NULL + value instead of raising an error if the conversion cannot be performed. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Name of the column or expression. - A column that evaluates to an array. - asc : bool, optional - Whether to sort in ascending or descending order. If `asc` is True (default), - then the sorting is in ascending order. If False, then in descending order. - A column that evaluates to a boolean. Must be a constant. + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert binary values. + A column that evaluates to a string. Must be a constant. - Returns - ------- - :class:`~pyspark.sql.Column` - Sorted array. - Returns a column that evaluates to an array. + See Also + -------- + :meth:`pyspark.sql.functions.to_binary` Examples -------- - Example 1: Sorting an array in ascending order + Example 1: Convert string to a binary with encoding specified >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | [NULL, 1, 2, 3]| - +----------------------+ + >>> df = spark.createDataFrame([("abc",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e, sf.lit("utf-8")).alias('r')).collect() + [Row(r=b'abc')] - Example 2: Sorting an array in descending order + Example 2: Convert string to a timestamp without encoding specified >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) - >>> df.select(sf.sort_array(df.data, asc=False)).show() - +-----------------------+ - |sort_array(data, false)| - +-----------------------+ - | [3, 2, 1, NULL]| - +-----------------------+ + >>> df = spark.createDataFrame([("414243",)], ["e"]) + >>> df.select(sf.try_to_binary(df.e).alias('r')).collect() + [Row(r=b'ABC')] - Example 3: Sorting an array with a single element + Example 3: Converion failure results in NULL when ANSI mode is on >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([([1],)], ['data']) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | [1]| - +----------------------+ - - Example 4: Sorting an empty array - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType - >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | []| - +----------------------+ - - Example 5: Sorting an array with null values - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField - >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) - >>> df = spark.createDataFrame([([None, None, None],)], schema=schema) - >>> df.select(sf.sort_array(df.data)).show() - +----------------------+ - |sort_array(data, true)| - +----------------------+ - | [NULL, NULL, NULL]| - +----------------------+ + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.range(1) + ... df.select(sf.try_to_binary(sf.lit("malformed"), sf.lit("hex"))).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +-----------------------------+ + |try_to_binary(malformed, hex)| + +-----------------------------+ + | NULL| + +-----------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("sort_array", _to_java_column(col), _enum_to_value(asc)) + if format is not None: + return _invoke_function_over_columns("try_to_binary", col, format) + else: + return _invoke_function_over_columns("try_to_binary", col) @_try_remote_functions -def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> Column: +def try_to_number(col: "ColumnOrName", format: "ColumnOrName") -> Column: """ - Array function: Generates a random permutation of the given array. - - .. versionadded:: 2.4.0 + Convert string 'col' to a number based on the string format `format`. Returns NULL if the + string 'col' does not match the expected format. The format follows the same semantics as the + to_number function. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or str - The name of the column or expression to be shuffled. - A column that evaluates to an array. - seed : :class:`~pyspark.sql.Column` or int, optional - Seed value for the random generator. - A column that evaluates to an integer or long. Must be a constant. - - .. versionadded:: 4.0.0 - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains an array of elements in random order. - Returns a column that evaluates to an array. + Input column or strings. + A column that evaluates to a string. + format : :class:`~pyspark.sql.Column` or str, optional + format to use to convert number values. + A column that evaluates to a string. Must be a constant. - Notes - ----- - The `shuffle` function is non-deterministic, meaning the order of the output array - can be different for each execution. + See Also + -------- + :meth:`pyspark.sql.functions.to_number` Examples -------- - Example 1: Shuffling a simple array - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 20, 3, 5) AS data") - >>> df.select("*", sf.shuffle(df.data, sf.lit(123))).show() # doctest: +SKIP - +-------------+-------------+ - | data|shuffle(data)| - +-------------+-------------+ - |[1, 20, 3, 5]|[5, 1, 20, 3]| - +-------------+-------------+ - - Example 2: Shuffling an array with null values - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 20, NULL, 5) AS data") - >>> df.select("*", sf.shuffle(sf.col("data"), 234)).show() # doctest: +SKIP - +----------------+----------------+ - | data| shuffle(data)| - +----------------+----------------+ - |[1, 20, NULL, 5]|[NULL, 5, 20, 1]| - +----------------+----------------+ - - Example 3: Shuffling an array with duplicate values + Example 1: Convert a string to a number with a format specified >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") - >>> df.select("*", sf.shuffle("data", 345)).show() # doctest: +SKIP - +------------------+------------------+ - | data| shuffle(data)| - +------------------+------------------+ - |[1, 2, 2, 3, 3, 3]|[2, 3, 3, 1, 2, 3]| - +------------------+------------------+ + >>> df = spark.createDataFrame([("$78.12",)], ["e"]) + >>> df.select(sf.try_to_number(df.e, sf.lit("$99.99")).alias('r')).show() + +-----+ + | r| + +-----+ + |78.12| + +-----+ - Example 4: Shuffling an array with random seed + Example 2: Converion failure results in NULL when ANSI mode is on >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") - >>> df.select("*", sf.shuffle("data")).show() # doctest: +SKIP - +------------------+------------------+ - | data| shuffle(data)| - +------------------+------------------+ - |[1, 2, 2, 3, 3, 3]|[3, 3, 2, 3, 2, 1]| - +------------------+------------------+ + >>> origin = spark.conf.get("spark.sql.ansi.enabled") + >>> spark.conf.set("spark.sql.ansi.enabled", "true") + >>> try: + ... df = spark.range(1) + ... df.select(sf.try_to_number(sf.lit("77"), sf.lit("$99.99")).alias('r')).show() + ... finally: + ... spark.conf.set("spark.sql.ansi.enabled", origin) + +----+ + | r| + +----+ + |NULL| + +----+ """ - if seed is not None: - return _invoke_function_over_columns("shuffle", col, lit(seed)) - else: - return _invoke_function_over_columns("shuffle", col) + return _invoke_function_over_columns("try_to_number", col, format) @_try_remote_functions -def flatten(col: "ColumnOrName") -> Column: +def contains(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Array function: creates a single array from an array of arrays. - If a structure of nested arrays is deeper than two levels, - only one level of nesting is removed. - - .. versionadded:: 2.4.0 + Returns a boolean. The value is True if right is found inside left. + Returns NULL if either input expression is NULL. Otherwise, returns False. + Both left or right must be of STRING or BINARY type. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or expression to be flattened. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains the flattened array. + left : :class:`~pyspark.sql.Column` or str + The input to check; may be NULL. + A column that evaluates to a string or binary. + right : :class:`~pyspark.sql.Column` or str + The value to find; may be NULL. + A column that evaluates to a string or binary. Examples -------- - Example 1: Flattening a simple nested array - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[1, 2, 3], [4, 5], [6]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show() - +------------------+ - | flatten(data)| - +------------------+ - |[1, 2, 3, 4, 5, 6]| - +------------------+ + >>> df = spark.createDataFrame([("Spark SQL", "Spark")], ['a', 'b']) + >>> df.select(contains(df.a, df.b).alias('r')).collect() + [Row(r=True)] - Example 2: Flattening an array with null values + >>> df = spark.createDataFrame([("414243", "4243",)], ["c", "d"]) + >>> df = df.select(to_binary("c").alias("c"), to_binary("d").alias("d")) + >>> df.printSchema() + root + |-- c: binary (nullable = true) + |-- d: binary (nullable = true) + >>> df.select(contains("c", "d"), contains("d", "c")).show() + +--------------+--------------+ + |contains(c, d)|contains(d, c)| + +--------------+--------------+ + | true| false| + +--------------+--------------+ + """ + return _invoke_function_over_columns("contains", left, right) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([None, [4, 5]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show() - +-------------+ - |flatten(data)| - +-------------+ - | NULL| - +-------------+ - Example 3: Flattening an array with more than two levels of nesting +@_try_remote_functions +def elt(*inputs: "ColumnOrName") -> Column: + """ + Returns the `n`-th input, e.g., returns `input2` when `n` is 2. + The function returns NULL if the index exceeds the length of the array + and `spark.sql.ansi.enabled` is set to false. If `spark.sql.ansi.enabled` is set to true, + it throws ArrayIndexOutOfBoundsException for invalid indices. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([[[1, 2], [3, 4]], [[5, 6], [7, 8]]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show(truncate=False) - +--------------------------------+ - |flatten(data) | - +--------------------------------+ - |[[1, 2], [3, 4], [5, 6], [7, 8]]| - +--------------------------------+ + .. versionadded:: 3.5.0 - Example 4: Flattening an array with mixed types + Parameters + ---------- + inputs : :class:`~pyspark.sql.Column` or str + Input columns or strings. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([['a', 'b', 'c'], [1, 2, 3]],)], ['data']) - >>> df.select(sf.flatten(df.data)).show() - +------------------+ - | flatten(data)| - +------------------+ - |[a, b, c, 1, 2, 3]| - +------------------+ + Examples + -------- + >>> df = spark.createDataFrame([(1, "scala", "java")], ['a', 'b', 'c']) + >>> df.select(elt(df.a, df.b, df.c).alias('r')).collect() + [Row(r='scala')] """ - return _invoke_function_over_columns("flatten", col) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + sc = _get_active_spark_context() + return _invoke_function("elt", _to_seq(sc, inputs, _to_java_column)) @_try_remote_functions -def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Column: +def find_in_set(str: "ColumnOrName", str_array: "ColumnOrName") -> Column: """ - Array function: creates an array containing a column repeated count times. + Returns the index (1-based) of the given string (`str`) in the comma-delimited + list (`strArray`). Returns 0, if the string was not found or if the given string (`str`) + contains a comma. - .. versionadded:: 2.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the element to be repeated. - A column of any type. - count : :class:`~pyspark.sql.Column` or str or int - The name of the column, an expression, - or an integer that represents the number of times to repeat the element. - A column that evaluates to an integer. - - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains an array of repeated elements. - Returns a column that evaluates to an array. - - See Also - -------- - :meth:`pyspark.sql.functions.array` + str : :class:`~pyspark.sql.Column` or str + The given string to be found. + A column that evaluates to a string. + str_array : :class:`~pyspark.sql.Column` or str + The comma-delimited list. + A column that evaluates to a string. Examples -------- - Example 1: Usage with string + >>> df = spark.createDataFrame([("ab", "abc,b,ab,c,def")], ['a', 'b']) + >>> df.select(find_in_set(df.a, df.b).alias('r')).collect() + [Row(r=3)] + """ + return _invoke_function_over_columns("find_in_set", str, str_array) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('ab',)], ['data']) - >>> df.select(sf.array_repeat(df.data, 3)).show() - +---------------------+ - |array_repeat(data, 3)| - +---------------------+ - | [ab, ab, ab]| - +---------------------+ - Example 2: Usage with integer +@_try_remote_functions +def like( + str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None +) -> Column: + """ + Returns true if str matches `pattern` with `escape`, + null if any arguments are null, false otherwise. + The default escape character is the '\'. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(3,)], ['data']) - >>> df.select(sf.array_repeat(df.data, 2)).show() - +---------------------+ - |array_repeat(data, 2)| - +---------------------+ - | [3, 3]| - +---------------------+ + .. versionadded:: 3.5.0 - Example 3: Usage with array + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + A string. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or str + A string. The pattern is a string which is matched literally, with + exception to the following special symbols: + _ matches any one character in the input (similar to . in posix regular expressions) + % matches zero or more characters in the input (similar to .* in posix regular + expressions) + Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order + to match "\abc", the pattern should be "\\abc". + When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back + to Spark 1.6 behavior regarding string literal parsing. For example, if the config is + enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. + escapeChar : :class:`~pyspark.sql.Column`, optional + An character added since Spark 3.0. The default escape character is the '\'. + If an escape character precedes a special symbol or another escape character, the + following character is matched literally. It is invalid to escape any other character. + A column that evaluates to a string. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(['apple', 'banana'],)], ['data']) - >>> df.select(sf.array_repeat(df.data, 2)).show(truncate=False) - +----------------------------------+ - |array_repeat(data, 2) | - +----------------------------------+ - |[[apple, banana], [apple, banana]]| - +----------------------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.ilike` + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.regexp_like` - Example 4: Usage with null + Examples + -------- + >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) + >>> df.select(like(df.a, df.b).alias('r')).collect() + [Row(r=True)] - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", IntegerType(), True) - ... ]) - >>> df = spark.createDataFrame([(None, )], schema=schema) - >>> df.select(sf.array_repeat(df.data, 3)).show() - +---------------------+ - |array_repeat(data, 3)| - +---------------------+ - | [NULL, NULL, NULL]| - +---------------------+ + >>> df = spark.createDataFrame( + ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], + ... ['a', 'b'] + ... ) + >>> df.select(like(df.a, df.b, lit('/')).alias('r')).collect() + [Row(r=True)] """ - count = _enum_to_value(count) - count = lit(count) if isinstance(count, int) else count - - return _invoke_function_over_columns("array_repeat", col, count) + if escapeChar is not None: + return _invoke_function_over_columns("like", str, pattern, escapeChar) + else: + return _invoke_function_over_columns("like", str, pattern) @_try_remote_functions -def arrays_zip(*cols: "ColumnOrName") -> Column: +def ilike( + str: "ColumnOrName", pattern: "ColumnOrName", escapeChar: Optional["Column"] = None +) -> Column: """ - Array function: Returns a merged array of structs in which the N-th struct contains all - N-th values of input arrays. If one of the arrays is shorter than others then - the resulting struct type value will be a `null` for missing elements. - - .. versionadded:: 2.4.0 + Returns true if str matches `pattern` with `escape` case-insensitively, + null if any arguments are null, false otherwise. + The default escape character is the '\'. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - Columns of arrays to be merged. - A column that evaluates to an array. + str : :class:`~pyspark.sql.Column` or str + A string. + A column that evaluates to a string. + pattern : :class:`~pyspark.sql.Column` or str + A string. The pattern is a string which is matched literally, with + exception to the following special symbols: + _ matches any one character in the input (similar to . in posix regular expressions) + % matches zero or more characters in the input (similar to .* in posix regular + expressions) + Since Spark 2.0, string literals are unescaped in our SQL parser. For example, in order + to match "\abc", the pattern should be "\\abc". + When SQL config 'spark.sql.parser.escapedStringLiterals' is enabled, it falls back + to Spark 1.6 behavior regarding string literal parsing. For example, if the config is + enabled, the pattern to match "\abc" should be "\abc". + A column that evaluates to a string. + escapeChar : :class:`~pyspark.sql.Column`, optional + An character added since Spark 3.0. The default escape character is the '\'. + If an escape character precedes a special symbol or another escape character, the + following character is matched literally. It is invalid to escape any other character. + A column that evaluates to a string. - Returns - ------- - :class:`~pyspark.sql.Column` - Merged array of entries. - Returns a column that evaluates to an array. + See Also + -------- + :meth:`pyspark.sql.functions.like` + :meth:`pyspark.sql.functions.rlike` + :meth:`pyspark.sql.functions.regexp` + :meth:`pyspark.sql.functions.regexp_like` Examples -------- - Example 1: Zipping two arrays of the same length - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, 3], ['a', 'b', 'c'])], ['nums', 'letters']) - >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) - +-------------------------+ - |arrays_zip(nums, letters)| - +-------------------------+ - |[{1, a}, {2, b}, {3, c}] | - +-------------------------+ - - - Example 2: Zipping arrays of different lengths - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2], ['a', 'b', 'c'])], ['nums', 'letters']) - >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) - +---------------------------+ - |arrays_zip(nums, letters) | - +---------------------------+ - |[{1, a}, {2, b}, {NULL, c}]| - +---------------------------+ - - Example 3: Zipping more than two arrays + >>> df = spark.createDataFrame([("Spark", "_park")], ['a', 'b']) + >>> df.select(ilike(df.a, df.b).alias('r')).collect() + [Row(r=True)] - >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame( - ... [([1, 2], ['a', 'b'], [True, False])], ['nums', 'letters', 'bools']) - >>> df.select(sf.arrays_zip(df.nums, df.letters, df.bools)).show(truncate=False) - +--------------------------------+ - |arrays_zip(nums, letters, bools)| - +--------------------------------+ - |[{1, a, true}, {2, b, false}] | - +--------------------------------+ - - Example 4: Zipping arrays with null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([([1, 2, None], ['a', None, 'c'])], ['nums', 'letters']) - >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) - +------------------------------+ - |arrays_zip(nums, letters) | - +------------------------------+ - |[{1, a}, {2, NULL}, {NULL, c}]| - +------------------------------+ + ... [("%SystemDrive%/Users/John", "/%SystemDrive/%//Users%")], + ... ['a', 'b'] + ... ) + >>> df.select(ilike(df.a, df.b, lit('/')).alias('r')).collect() + [Row(r=True)] """ - return _invoke_function_over_seq_of_columns("arrays_zip", cols) + if escapeChar is not None: + return _invoke_function_over_columns("ilike", str, pattern, escapeChar) + else: + return _invoke_function_over_columns("ilike", str, pattern) @_try_remote_functions -def sequence( - start: "ColumnOrName", stop: "ColumnOrName", step: Optional["ColumnOrName"] = None -) -> Column: +def lcase(str: "ColumnOrName") -> Column: """ - Array function: Generate a sequence of integers from `start` to `stop`, incrementing by `step`. - If `step` is not set, the function increments by 1 if `start` is less than or equal to `stop`, - otherwise it decrements by 1. - - .. versionadded:: 2.4.0 + Returns `str` with all characters changed to lowercase. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - start : :class:`~pyspark.sql.Column` or str - The starting value (inclusive) of the sequence. - A column that evaluates to an integral, date, or timestamp. - stop : :class:`~pyspark.sql.Column` or str - The last value (inclusive) of the sequence. - A column that evaluates to an integral, date, or timestamp. - step : :class:`~pyspark.sql.Column` or str, optional - The value to add to the current element to get the next element in the sequence. - The default is 1 if `start` is less than or equal to `stop`, otherwise -1. - A column that evaluates to an integral or interval. + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. - Returns - ------- - :class:`~pyspark.sql.Column` - A new column that contains an array of sequence values. - Returns a column that evaluates to an array. + See Also + -------- + :meth:`pyspark.sql.functions.lower` + :meth:`pyspark.sql.functions.ucase` + :meth:`pyspark.sql.functions.upper` Examples -------- - Example 1: Generating a sequence with default step - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(-2, 2)], ['start', 'stop']) - >>> df.select(sf.sequence(df.start, df.stop)).show() - +---------------------+ - |sequence(start, stop)| - +---------------------+ - | [-2, -1, 0, 1, 2]| - +---------------------+ + >>> spark.range(1).select(sf.lcase(sf.lit("Spark"))).show() + +------------+ + |lcase(Spark)| + +------------+ + | spark| + +------------+ + """ + return _invoke_function_over_columns("lcase", str) - Example 2: Generating a sequence with a custom step - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(4, -4, -2)], ['start', 'stop', 'step']) - >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() - +---------------------------+ - |sequence(start, stop, step)| - +---------------------------+ - | [4, 2, 0, -2, -4]| - +---------------------------+ +@_try_remote_functions +def ucase(str: "ColumnOrName") -> Column: + """ + Returns `str` with all characters changed to uppercase. + .. versionadded:: 3.5.0 - Example 3: Generating a sequence with a negative step + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.upper` + :meth:`pyspark.sql.functions.lcase` + :meth:`pyspark.sql.functions.lower` + Examples + -------- >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(5, 1, -1)], ['start', 'stop', 'step']) - >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() - +---------------------------+ - |sequence(start, stop, step)| - +---------------------------+ - | [5, 4, 3, 2, 1]| - +---------------------------+ + >>> spark.range(1).select(sf.ucase(sf.lit("Spark"))).show() + +------------+ + |ucase(Spark)| + +------------+ + | SPARK| + +------------+ """ - if step is None: - return _invoke_function_over_columns("sequence", start, stop) - else: - return _invoke_function_over_columns("sequence", start, stop, step) - + return _invoke_function_over_columns("ucase", str) -# ---------------------- Struct Functions ---------------------- +@_try_remote_functions +def left(str: "ColumnOrName", len: "ColumnOrName") -> Column: + """ + Returns the leftmost `len`(`len` can be string type) characters from the string `str`, + if `len` is less or equal than 0 the result is an empty string. -@overload -def struct(*cols: "ColumnOrName") -> Column: ... + .. versionadded:: 3.5.0 + Parameters + ---------- + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string or binary. + len : :class:`~pyspark.sql.Column` or str + Input column or strings, the leftmost `len`. + A column that evaluates to an integer. -@overload -def struct(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... + Examples + -------- + >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) + >>> df.select(left(df.a, df.b).alias('r')).collect() + [Row(r='Spa')] + """ + return _invoke_function_over_columns("left", str, len) @_try_remote_functions -def struct( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], -) -> Column: - """Creates a new struct column. - - .. versionadded:: 1.4.0 +def right(str: "ColumnOrName", len: "ColumnOrName") -> Column: + """ + Returns the rightmost `len`(`len` can be string type) characters from the string `str`, + if `len` is less or equal than 0 the result is an empty string. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - cols : list, set, :class:`~pyspark.sql.Column` or column name - column names or :class:`~pyspark.sql.Column`\\s to contain in the output struct. - Each a column of any type. - - Returns - ------- - :class:`~pyspark.sql.Column` - a struct type column of given columns. - Returns a column that evaluates to a struct. - - See Also - -------- - :meth:`pyspark.sql.functions.named_struct` + str : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + len : :class:`~pyspark.sql.Column` or str + Input column or strings, the rightmost `len`. + A column that evaluates to an integer. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) - >>> df.select("*", sf.struct('age', df.name)).show() - +-----+---+-----------------+ - | name|age|struct(age, name)| - +-----+---+-----------------+ - |Alice| 2| {2, Alice}| - | Bob| 5| {5, Bob}| - +-----+---+-----------------+ + >>> df = spark.createDataFrame([("Spark SQL", 3,)], ['a', 'b']) + >>> df.select(right(df.a, df.b).alias('r')).collect() + [Row(r='SQL')] """ - if len(cols) == 1 and isinstance(cols[0], (list, set)): - cols = cols[0] # type: ignore[assignment] - return _invoke_function_over_seq_of_columns("struct", cols) # type: ignore[arg-type] + return _invoke_function_over_columns("right", str, len) @_try_remote_functions -def named_struct(*cols: "ColumnOrName") -> Column: +def mask( + col: "ColumnOrName", + upperChar: Optional["ColumnOrName"] = None, + lowerChar: Optional["ColumnOrName"] = None, + digitChar: Optional["ColumnOrName"] = None, + otherChar: Optional["ColumnOrName"] = None, +) -> Column: """ - Creates a struct with the given field names and values. + Masks the given string value. This can be useful for creating copies of tables with sensitive + information removed. .. versionadded:: 3.5.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - list of columns to work on. + col: :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to a string. + upperChar: :class:`~pyspark.sql.Column` or str, optional + character to replace upper-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. + lowerChar: :class:`~pyspark.sql.Column` or str, optional + character to replace lower-case characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. + digitChar: :class:`~pyspark.sql.Column` or str, optional + character to replace digit characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. + otherChar: :class:`~pyspark.sql.Column` or str, optional + character to replace all other characters with. Specify NULL to retain original character. + A column that evaluates to a string. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - - See Also - -------- - :meth:`pyspark.sql.functions.struct` + Returns a column that evaluates to a string. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, 2)], ['a', 'b']) - >>> df.select("*", sf.named_struct(sf.lit('x'), df.a, sf.lit('y'), "b")).show() - +---+---+------------------------+ - | a| b|named_struct(x, a, y, b)| - +---+---+------------------------+ - | 1| 2| {1, 2}| - +---+---+------------------------+ + >>> df = spark.createDataFrame([("AbCD123-@$#",), ("abcd-EFGH-8765-4321",)], ['data']) + >>> df.select(mask(df.data).alias('r')).collect() + [Row(r='XxXXnnn-@$#'), Row(r='xxxx-XXXX-nnnn-nnnn')] + >>> df.select(mask(df.data, lit('Y')).alias('r')).collect() + [Row(r='YxYYnnn-@$#'), Row(r='xxxx-YYYY-nnnn-nnnn')] + >>> df.select(mask(df.data, lit('Y'), lit('y')).alias('r')).collect() + [Row(r='YyYYnnn-@$#'), Row(r='yyyy-YYYY-nnnn-nnnn')] + >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d')).alias('r')).collect() + [Row(r='YyYYddd-@$#'), Row(r='yyyy-YYYY-dddd-dddd')] + >>> df.select(mask(df.data, lit('Y'), lit('y'), lit('d'), lit('*')).alias('r')).collect() + [Row(r='YyYYddd****'), Row(r='yyyy*YYYY*dddd*dddd')] """ - return _invoke_function_over_seq_of_columns("named_struct", cols) + _upperChar = lit("X") if upperChar is None else upperChar + _lowerChar = lit("x") if lowerChar is None else lowerChar + _digitChar = lit("n") if digitChar is None else digitChar + _otherChar = lit(None) if otherChar is None else otherChar + return _invoke_function_over_columns( + "mask", col, _upperChar, _lowerChar, _digitChar, _otherChar + ) -# ---------------------- Map Functions ---------------------- +@_try_remote_functions +def collate(col: "ColumnOrName", collation: str) -> Column: + """ + Marks a given column with specified collation. -@overload -def create_map(*cols: "ColumnOrName") -> Column: ... + .. versionadded:: 4.0.0 + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + Target string column to work on. + collation : str + Target collation name. -@overload -def create_map(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... + Returns + ------- + :class:`~pyspark.sql.Column` + A new column of string type, where each value has the specified collation. + """ + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("collate", _to_java_column(col), _enum_to_value(collation)) @_try_remote_functions -def create_map( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], -) -> Column: +def collation(col: "ColumnOrName") -> Column: """ - Map function: Creates a new map column from an even number of input columns or - column references. The input columns are grouped into key-value pairs to form a map. - For instance, the input (key1, value1, key2, value2, ...) would produce a map that - associates key1 with value1, key2 with value2, and so on. The function supports - grouping columns as a list as well. - - .. versionadded:: 2.0.0 + Returns the collation name of a given column. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - The input column names or :class:`~pyspark.sql.Column` objects grouped into - key-value pairs. These can also be expressed as a list of columns. + col : :class:`~pyspark.sql.Column` or str + Target string column to work on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new Column of Map type, where each value is a map formed from the corresponding - key-value pairs provided in the input arguments. - + collation name of a given expression. + Returns a column that evaluates to a string. + + Examples + -------- + >>> df = spark.createDataFrame([('name',)], ['dt']) + >>> df.select(collation('dt').alias('collation')).show(truncate=False) + +--------------------------+ + |collation | + +--------------------------+ + |SYSTEM.BUILTIN.UTF8_BINARY| + +--------------------------+ + """ + return _invoke_function_over_columns("collation", col) + + +@_try_remote_functions +def quote(col: "ColumnOrName") -> Column: + r"""Returns `str` enclosed by single quotes and each instance of + single quote in it is preceded by a backslash. + + .. versionadded:: 4.1.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + target column to be quoted. + A column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + quoted string + Returns a column that evaluates to a string. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame(["Don't"], "STRING") + >>> df.select("*", sf.quote("value")).show() + +-----+------------+ + |value|quote(value)| + +-----+------------+ + |Don't| 'Don\'t'| + +-----+------------+ + """ + return _invoke_function_over_columns("quote", col) + + +# ---------------------- Collection functions ------------------------------ + + +@overload +def create_map(*cols: "ColumnOrName") -> Column: ... + + +@overload +def create_map(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... + + +@_try_remote_functions +def create_map( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], +) -> Column: + """ + Map function: Creates a new map column from an even number of input columns or + column references. The input columns are grouped into key-value pairs to form a map. + For instance, the input (key1, value1, key2, value2, ...) would produce a map that + associates key1 with value1, key2 with value2, and so on. The function supports + grouping columns as a list as well. + + .. versionadded:: 2.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + cols : :class:`~pyspark.sql.Column` or str + The input column names or :class:`~pyspark.sql.Column` objects grouped into + key-value pairs. These can also be expressed as a list of columns. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new Column of Map type, where each value is a map formed from the corresponding + key-value pairs provided in the input arguments. + Examples -------- Example 1: Basic usage of create_map function. @@ -19774,147 +20113,119 @@ def map_from_arrays(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: return _invoke_function_over_columns("map_from_arrays", col1, col2) -@_try_remote_functions -def map_contains_key(col: "ColumnOrName", value: Any) -> Column: - """ - Map function: Returns true if the map contains the key. - - .. versionadded:: 3.4.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - The name of the column or an expression that represents the map. - value : - A literal value, or a :class:`~pyspark.sql.Column` expression. - - .. versionchanged:: 4.0.0 - `value` now also accepts a Column type. - - Returns - ------- - :class:`~pyspark.sql.Column` - True if key is in the map and False otherwise. - - Examples - -------- - Example 1: The key is in the map - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.map_contains_key("data", 1)).show() - +-------------------------+ - |map_contains_key(data, 1)| - +-------------------------+ - | true| - +-------------------------+ - - Example 2: The key is not in the map - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.map_contains_key("data", -1)).show() - +--------------------------+ - |map_contains_key(data, -1)| - +--------------------------+ - | false| - +--------------------------+ +@overload +def array(*cols: "ColumnOrName") -> Column: ... - Example 3: Check for key using a column - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data, 1 as key") - >>> df.select(sf.map_contains_key("data", sf.col("key"))).show() - +---------------------------+ - |map_contains_key(data, key)| - +---------------------------+ - | true| - +---------------------------+ - """ - return _invoke_function_over_columns("map_contains_key", col, lit(value)) +@overload +def array(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... @_try_remote_functions -def map_keys(col: "ColumnOrName") -> Column: +def array( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], +) -> Column: """ - Map function: Returns an unordered array containing the keys of the map. + Collection function: Creates a new array column from the input columns or column names. - .. versionadded:: 2.3.0 + .. versionadded:: 1.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of column or expression + cols : :class:`~pyspark.sql.Column` or str + Column names or :class:`~pyspark.sql.Column` objects that have the same data type. Returns ------- :class:`~pyspark.sql.Column` - Keys of the map as an array. + A new Column of array type, where each value is an array containing the corresponding values + from the input columns. + Returns a column that evaluates to an array. Examples -------- - Example 1: Extracting keys from a simple map + Example 1: Basic usage of array function with column names. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_keys("data"))).show() - +--------------------------------+ - |sort_array(map_keys(data), true)| - +--------------------------------+ - | [1, 2]| - +--------------------------------+ + >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array('name', 'occupation')).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, doctor]| + | [Bob, engineer]| + +-----------------------+ - Example 2: Extracting keys from a map with complex keys + Example 2: Usage of array function with Column objects. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(array(1, 2), 'a', array(3, 4), 'b') as data") - >>> df.select(sf.sort_array(sf.map_keys("data"))).show() - +--------------------------------+ - |sort_array(map_keys(data), true)| - +--------------------------------+ - | [[1, 2], [3, 4]]| - +--------------------------------+ + >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array(df.name, df.occupation)).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, doctor]| + | [Bob, engineer]| + +-----------------------+ - Example 3: Extracting keys from a map with duplicate keys + Example 3: Single argument as list of column names. >>> from pyspark.sql import functions as sf - >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") - >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") - >>> df.select(sf.map_keys("data")).show() - +--------------+ - |map_keys(data)| - +--------------+ - | [1]| - +--------------+ - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) + >>> df = spark.createDataFrame([("Alice", "doctor"), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array(['name', 'occupation'])).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, doctor]| + | [Bob, engineer]| + +-----------------------+ - Example 4: Extracting keys from an empty map + Example 4: Usage of array function with columns of different types. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map() as data") - >>> df.select(sf.map_keys("data")).show() - +--------------+ - |map_keys(data)| - +--------------+ - | []| - +--------------+ + >>> df = spark.createDataFrame( + ... [("Alice", 2, 22.2), ("Bob", 5, 36.1)], + ... ("name", "age", "weight")) + >>> df.select(sf.array(['age', 'weight'])).show() + +------------------+ + |array(age, weight)| + +------------------+ + | [2.0, 22.2]| + | [5.0, 36.1]| + +------------------+ + + Example 5: array function with a column containing null values. + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("Alice", None), ("Bob", "engineer")], + ... ("name", "occupation")) + >>> df.select(sf.array('name', 'occupation')).show() + +-----------------------+ + |array(name, occupation)| + +-----------------------+ + | [Alice, NULL]| + | [Bob, engineer]| + +-----------------------+ """ - return _invoke_function_over_columns("map_keys", col) + if len(cols) == 1 and isinstance(cols[0], (list, set)): + cols = cols[0] # type: ignore[assignment] + return _invoke_function_over_seq_of_columns("array", cols) # type: ignore[arg-type] @_try_remote_functions -def map_values(col: "ColumnOrName") -> Column: +def array_contains(col: "ColumnOrName", value: Any) -> Column: """ - Map function: Returns an unordered array containing the values of the map. + Collection function: Returns true if the array contains the value, false if not. Returns + null if the array or value is null, or if the value is not found and the array contains a + null element. - .. versionadded:: 2.3.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -19922,152 +20233,176 @@ def map_values(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or str - Name of column or expression + The target column containing the arrays. + A column that evaluates to an array. + value : + The value or column to check for in the array. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` - Values of the map as an array. + A new Column of Boolean type, where each value indicates whether the corresponding array + from the input column contains the specified value. + Returns a column that evaluates to a boolean. + + See Also + -------- + :meth:`pyspark.sql.functions.array_position` Examples -------- - Example 1: Extracting values from a simple map + Example 1: Basic usage of array_contains function. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_values("data"))).show() - +----------------------------------+ - |sort_array(map_values(data), true)| - +----------------------------------+ - | [a, b]| - +----------------------------------+ + >>> df = spark.createDataFrame([(["a", "b", "c"],), ([],)], ['data']) + >>> df.select(sf.array_contains(df.data, "a")).show() + +-----------------------+ + |array_contains(data, a)| + +-----------------------+ + | true| + | false| + +-----------------------+ - Example 2: Extracting values from a map with complex values + Example 2: Usage of array_contains function with a column. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, array('a', 'b'), 2, array('c', 'd')) as data") - >>> df.select(sf.sort_array(sf.map_values("data"))).show() - +----------------------------------+ - |sort_array(map_values(data), true)| - +----------------------------------+ - | [[a, b], [c, d]]| - +----------------------------------+ + >>> df = spark.createDataFrame([(["a", "b", "c"], "c"), + ... (["c", "d", "e"], "d"), + ... (["e", "a", "c"], "b")], ["data", "item"]) + >>> df.select(sf.array_contains(df.data, sf.col("item"))).show() + +--------------------------+ + |array_contains(data, item)| + +--------------------------+ + | true| + | true| + | false| + +--------------------------+ - Example 3: Extracting values from a map with null values + Example 3: Attempt to use array_contains function with a null array. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, null, 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_values("data"))).show() - +----------------------------------+ - |sort_array(map_values(data), true)| - +----------------------------------+ - | [NULL, b]| - +----------------------------------+ + >>> df = spark.createDataFrame([(None,), (["a", "b", "c"],)], ['data']) + >>> df.select(sf.array_contains(df.data, "a")).show() + +-----------------------+ + |array_contains(data, a)| + +-----------------------+ + | NULL| + | true| + +-----------------------+ - Example 4: Extracting values from a map with duplicate values + Example 4: Usage of array_contains with an array column containing null values. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'a') as data") - >>> df.select(sf.map_values("data")).show() - +----------------+ - |map_values(data)| - +----------------+ - | [a, a]| - +----------------+ + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_contains(df.data, "a")).show() + +-----------------------+ + |array_contains(data, a)| + +-----------------------+ + | true| + +-----------------------+ - Example 5: Extracting values from an empty map + Example 5: Value absent from an array that contains a null element returns NULL. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map() as data") - >>> df.select(sf.map_values("data")).show() - +----------------+ - |map_values(data)| - +----------------+ - | []| - +----------------+ + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_contains(df.data, "b")).show() + +-----------------------+ + |array_contains(data, b)| + +-----------------------+ + | NULL| + +-----------------------+ """ - return _invoke_function_over_columns("map_values", col) + return _invoke_function_over_columns("array_contains", col, lit(value)) @_try_remote_functions -def map_entries(col: "ColumnOrName") -> Column: +def arrays_overlap(a1: "ColumnOrName", a2: "ColumnOrName") -> Column: """ - Map function: Returns an unordered array of all entries in the given map. + Collection function: This function returns a boolean column indicating if the input arrays + have common non-null elements, returning true if they do, null if the arrays do not contain + any common elements but are not empty and at least one of them contains a null element, + and false otherwise. - .. versionadded:: 3.0.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 - Spark Connect. + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of column or expression + a1, a2 : :class:`~pyspark.sql.Column` or str + The names of the columns that contain the input arrays. + Each a column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - An array of key value pairs as a struct type + A new Column of Boolean type, where each value indicates whether the corresponding arrays + from the input columns contain any common elements. + Returns a column that evaluates to a boolean. Examples -------- - Example 1: Extracting entries from a simple map + Example 1: Basic usage of arrays_overlap function. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") - >>> df.select(sf.sort_array(sf.map_entries("data"))).show() - +-----------------------------------+ - |sort_array(map_entries(data), true)| - +-----------------------------------+ - | [{1, a}, {2, b}]| - +-----------------------------------+ + >>> df = spark.createDataFrame([(["a", "b"], ["b", "c"]), (["a"], ["b", "c"])], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | true| + | false| + +--------------------+ - Example 2: Extracting entries from a map with complex keys and values + Example 2: Usage of arrays_overlap function with arrays containing null elements. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(array(1, 2), array('a', 'b'), " - ... "array(3, 4), array('c', 'd')) as data") - >>> df.select(sf.sort_array(sf.map_entries("data"))).show(truncate=False) - +------------------------------------+ - |sort_array(map_entries(data), true) | - +------------------------------------+ - |[{[1, 2], [a, b]}, {[3, 4], [c, d]}]| - +------------------------------------+ + >>> df = spark.createDataFrame([(["a", None], ["b", None]), (["a"], ["b", "c"])], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | NULL| + | false| + +--------------------+ - Example 3: Extracting entries from a map with duplicate keys + Example 3: Usage of arrays_overlap function with arrays that are null. >>> from pyspark.sql import functions as sf - >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") - >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") - >>> df.select(sf.map_entries("data")).show() - +-----------------+ - |map_entries(data)| - +-----------------+ - | [{1, b}]| - +-----------------+ - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) + >>> df = spark.createDataFrame([(None, ["b", "c"]), (["a"], None)], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | NULL| + | NULL| + +--------------------+ - Example 4: Extracting entries from an empty map + Example 4: Usage of arrays_overlap on arrays with identical elements. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map() as data") - >>> df.select(sf.map_entries("data")).show() - +-----------------+ - |map_entries(data)| - +-----------------+ - | []| - +-----------------+ + >>> df = spark.createDataFrame([(["a", "b"], ["a", "b"]), (["a"], ["a"])], ['x', 'y']) + >>> df.select(sf.arrays_overlap(df.x, df.y)).show() + +--------------------+ + |arrays_overlap(x, y)| + +--------------------+ + | true| + | true| + +--------------------+ """ - return _invoke_function_over_columns("map_entries", col) + return _invoke_function_over_columns("arrays_overlap", a1, a2) @_try_remote_functions -def map_from_entries(col: "ColumnOrName") -> Column: +def slice( + x: "ColumnOrName", start: Union["ColumnOrName", int], length: Union["ColumnOrName", int] +) -> Column: """ - Map function: Transforms an array of key-value pair entries (structs with two fields) - into a map. The first field of each entry is used as the key and the second field - as the value in the resulting map column + Array function: Returns a new array column by slicing the input array column from + a start index to a specific length. The indices start at 1, and can be negative to index + from the end of the array. The length specifies the number of elements in the resulting array. .. versionadded:: 2.4.0 @@ -20076,1623 +20411,1778 @@ def map_from_entries(col: "ColumnOrName") -> Column: Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - Name of column or expression + x : :class:`~pyspark.sql.Column` or str + Input array column or column name to be sliced. + A column that evaluates to an array. + start : :class:`~pyspark.sql.Column`, str, or int + The start index for the slice operation. If negative, starts the index from the + end of the array. + A column that evaluates to an integer. + length : :class:`~pyspark.sql.Column`, str, or int + The length of the slice, representing number of elements in the resulting array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A map created from the given array of entries. + A new Column object of Array type, where each value is a slice of the corresponding + list from the input column. + Returns a column that evaluates to an array. Examples -------- - Example 1: Basic usage of map_from_entries + Example 1: Basic usage of the slice function. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array(struct(1, 'a'), struct(2, 'b')) as data") - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {1 -> a, 2 -> b}| - +----------------------+ + >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) + >>> df.select(sf.slice(df.x, 2, 2)).show() + +--------------+ + |slice(x, 2, 2)| + +--------------+ + | [2, 3]| + | [5]| + +--------------+ - Example 2: map_from_entries with null values + Example 2: Slicing with negative start index. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array(struct(1, null), struct(2, 'b')) as data") - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {1 -> NULL, 2 -> b}| - +----------------------+ - - Example 3: map_from_entries with a DataFrame - - >>> from pyspark.sql import Row, functions as sf - >>> df = spark.createDataFrame([([Row(1, "a"), Row(2, "b")],), ([Row(3, "c")],)], ['data']) - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {1 -> a, 2 -> b}| - | {3 -> c}| - +----------------------+ + >>> df = spark.createDataFrame([([1, 2, 3],), ([4, 5],)], ['x']) + >>> df.select(sf.slice(df.x, -1, 1)).show() + +---------------+ + |slice(x, -1, 1)| + +---------------+ + | [3]| + | [5]| + +---------------+ - Example 4: map_from_entries with empty array + Example 3: Slice function with column inputs for start and length. >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StringType, IntegerType, StructType, StructField - >>> schema = StructType([ - ... StructField("data", ArrayType( - ... StructType([ - ... StructField("key", IntegerType()), - ... StructField("value", StringType()) - ... ]) - ... ), True) - ... ]) - >>> df = spark.createDataFrame([([],)], schema=schema) - >>> df.select(sf.map_from_entries(df.data)).show() - +----------------------+ - |map_from_entries(data)| - +----------------------+ - | {}| - +----------------------+ + >>> df = spark.createDataFrame([([1, 2, 3], 2, 2), ([4, 5], 1, 3)], ['x', 'start', 'length']) + >>> df.select(sf.slice(df.x, df.start, df.length)).show() + +-----------------------+ + |slice(x, start, length)| + +-----------------------+ + | [2, 3]| + | [4, 5]| + +-----------------------+ """ - return _invoke_function_over_columns("map_from_entries", col) - - -@overload -def map_concat(*cols: "ColumnOrName") -> Column: ... - + start = _enum_to_value(start) + start = lit(start) if isinstance(start, int) else start + length = _enum_to_value(length) + length = lit(length) if isinstance(length, int) else length -@overload -def map_concat(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... + return _invoke_function_over_columns("slice", x, start, length) @_try_remote_functions -def map_concat( - *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], -) -> Column: +def trim_array(x: "ColumnOrName", n: Union["ColumnOrName", int]) -> Column: """ - Map function: Returns the union of all given maps. - - .. versionadded:: 2.4.0 + Array function: Returns the given array column with the last ``n`` elements removed. + Raises an error if ``n`` is negative or greater than the number of elements in the array. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.4.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or str - Column names or :class:`~pyspark.sql.Column` + x : :class:`~pyspark.sql.Column` or str + Input array column or column name to be trimmed. + A column that evaluates to an array. + n : :class:`~pyspark.sql.Column`, str, or int + The number of elements to remove from the end of the array. Must be between 0 and + the number of elements in the array (inclusive). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A map of merged entries from other maps. - - Notes - ----- - For duplicate keys in input maps, the handling is governed by `spark.sql.mapKeyDedupPolicy`. - By default, it throws an exception. If set to `LAST_WIN`, it uses the last map's value. + A new Column object of Array type, where each value is the corresponding input array + with its last ``n`` elements removed. + Returns a column that evaluates to an array. Examples -------- - Example 1: Basic usage of map_concat - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +------------------------+ - |map_concat(map1, map2) | - +------------------------+ - |{1 -> a, 2 -> b, 3 -> c}| - +------------------------+ - - Example 2: map_concat with overlapping keys - - >>> from pyspark.sql import functions as sf - >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(2, 'c', 3, 'd') as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +------------------------+ - |map_concat(map1, map2) | - +------------------------+ - |{1 -> a, 2 -> c, 3 -> d}| - +------------------------+ - >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) - - Example 3: map_concat with three maps - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a') as map1, map(2, 'b') as map2, map(3, 'c') as map3") - >>> df.select(sf.map_concat("map1", "map2", "map3")).show(truncate=False) - +----------------------------+ - |map_concat(map1, map2, map3)| - +----------------------------+ - |{1 -> a, 2 -> b, 3 -> c} | - +----------------------------+ - - Example 4: map_concat with empty map + Example 1: Basic usage of the trim_array function. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map() as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +----------------------+ - |map_concat(map1, map2)| - +----------------------+ - |{1 -> a, 2 -> b} | - +----------------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 4, 5],), ([4, 5],)], ['x']) + >>> df.select(sf.trim_array(df.x, 2)).show() + +----------------+ + |trim_array(x, 2)| + +----------------+ + | [1, 2, 3]| + | []| + +----------------+ - Example 5: map_concat with null values + Example 2: trim_array function with a column input for n. >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, null) as map2") - >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) - +---------------------------+ - |map_concat(map1, map2) | - +---------------------------+ - |{1 -> a, 2 -> b, 3 -> NULL}| - +---------------------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 4, 5], 1), ([4, 5], 0)], ['x', 'n']) + >>> df.select(sf.trim_array(df.x, df.n)).show() + +----------------+ + |trim_array(x, n)| + +----------------+ + | [1, 2, 3, 4]| + | [4, 5]| + +----------------+ """ - if len(cols) == 1 and isinstance(cols[0], (list, set)): - cols = cols[0] # type: ignore[assignment] - return _invoke_function_over_seq_of_columns("map_concat", cols) # type: ignore[arg-type] + n = _enum_to_value(n) + n = lit(n) if isinstance(n, int) else n + return _invoke_function_over_columns("trim_array", x, n) @_try_remote_functions -def str_to_map( - text: "ColumnOrName", - pairDelim: Optional["ColumnOrName"] = None, - keyValueDelim: Optional["ColumnOrName"] = None, +def array_join( + col: "ColumnOrName", delimiter: str, null_replacement: Optional[str] = None ) -> Column: """ - Map function: Converts a string into a map after splitting the text into key/value pairs - using delimiters. Both `pairDelim` and `keyValueDelim` are treated as regular expressions. + Array function: Returns a string column by concatenating the elements of the input + array column using the delimiter. Null values within the array can be replaced with + a specified string through the null_replacement argument. If null_replacement is + not set, null values are ignored. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - text : :class:`~pyspark.sql.Column` or str - Input column or strings. - A column that evaluates to a string. - pairDelim : :class:`~pyspark.sql.Column` or str, optional - Delimiter to use to split pairs. Default is comma (,). + col : :class:`~pyspark.sql.Column` or str + The input column containing the arrays to be joined. + A column that evaluates to an array. + delimiter : str + The string to be used as the delimiter when joining the array elements. A column that evaluates to a string. - keyValueDelim : :class:`~pyspark.sql.Column` or str, optional - Delimiter to use to split key/value. Default is colon (:). + null_replacement : str, optional + The string to replace null values within the array. If not set, null values are ignored. A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new column of map type where each string in the original column is converted into a map. - Returns a column that evaluates to a map. + A new column of string type, where each value is the result of joining the corresponding + array from the input column. + Returns a column that evaluates to a string. + + See Also + -------- + :meth:`pyspark.sql.functions.concat` + :meth:`pyspark.sql.functions.concat_ws` Examples -------- - Example 1: Using default delimiters + Example 1: Basic usage of array_join function. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3",)], ["e"]) - >>> df.select(sf.str_to_map(df.e)).show(truncate=False) - +------------------------+ - |str_to_map(e, ,, :) | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - +------------------------+ + >>> df = spark.createDataFrame([(["a", "b", "c"],), (["a", "b"],)], ['data']) + >>> df.select(sf.array_join(df.data, ",")).show() + +-------------------+ + |array_join(data, ,)| + +-------------------+ + | a,b,c| + | a,b| + +-------------------+ - Example 2: Using custom delimiters + Example 2: Usage of array_join function with null_replacement argument. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a=1;b=2;c=3",)], ["e"]) - >>> df.select(sf.str_to_map(df.e, sf.lit(";"), sf.lit("="))).show(truncate=False) - +------------------------+ - |str_to_map(e, ;, =) | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - +------------------------+ + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_join(df.data, ",", "NULL")).show() + +-------------------------+ + |array_join(data, ,, NULL)| + +-------------------------+ + | a,NULL,c| + +-------------------------+ - Example 3: Using different delimiters for different rows + Example 3: Usage of array_join function without null_replacement argument. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3",), ("d=4;e=5;f=6",)], ["e"]) - >>> df.select(sf.str_to_map(df.e, - ... sf.when(df.e.contains(";"), sf.lit(";")).otherwise(sf.lit(",")), - ... sf.when(df.e.contains("="), sf.lit("=")).otherwise(sf.lit(":"))).alias("str_to_map") - ... ).show(truncate=False) - +------------------------+ - |str_to_map | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - |{d -> 4, e -> 5, f -> 6}| - +------------------------+ + >>> df = spark.createDataFrame([(["a", None, "c"],)], ['data']) + >>> df.select(sf.array_join(df.data, ",")).show() + +-------------------+ + |array_join(data, ,)| + +-------------------+ + | a,c| + +-------------------+ - Example 4: Using a column of delimiters + Example 4: Usage of array_join function with an array that is null. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3", ","), ("d=4;e=5;f=6", ";")], ["e", "delim"]) - >>> df.select(sf.str_to_map(df.e, df.delim, sf.lit(":"))).show(truncate=False) - +---------------------------------------+ - |str_to_map(e, delim, :) | - +---------------------------------------+ - |{a -> 1, b -> 2, c -> 3} | - |{d=4 -> NULL, e=5 -> NULL, f=6 -> NULL}| - +---------------------------------------+ + >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([(None,)], schema) + >>> df.select(sf.array_join(df.data, ",")).show() + +-------------------+ + |array_join(data, ,)| + +-------------------+ + | NULL| + +-------------------+ - Example 5: Using a column of key/value delimiters + Example 5: Usage of array_join function with an array containing only null values. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a:1,b:2,c:3", ":"), ("d=4;e=5;f=6", "=")], ["e", "delim"]) - >>> df.select(sf.str_to_map(df.e, sf.lit(","), df.delim)).show(truncate=False) - +------------------------+ - |str_to_map(e, ,, delim) | - +------------------------+ - |{a -> 1, b -> 2, c -> 3}| - |{d -> 4;e=5;f=6} | - +------------------------+ + >>> from pyspark.sql.types import StructType, StructField, ArrayType, StringType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([([None, None],)], schema) + >>> df.select(sf.array_join(df.data, ",", "NULL")).show() + +-------------------------+ + |array_join(data, ,, NULL)| + +-------------------------+ + | NULL,NULL| + +-------------------------+ """ - if pairDelim is None: - pairDelim = lit(",") - if keyValueDelim is None: - keyValueDelim = lit(":") - return _invoke_function_over_columns("str_to_map", text, pairDelim, keyValueDelim) - + from pyspark.sql.classic.column import _to_java_column -# ---------------------- Aggregate Functions ---------------------- + _get_active_spark_context() + if null_replacement is None: + return _invoke_function("array_join", _to_java_column(col), _enum_to_value(delimiter)) + else: + return _invoke_function( + "array_join", + _to_java_column(col), + _enum_to_value(delimiter), + _enum_to_value(null_replacement), + ) @_try_remote_functions -def try_avg(col: "ColumnOrName") -> Column: +def concat(*cols: "ColumnOrName") -> Column: """ - Returns the mean calculated from values of a group and the result is null on overflow. + Collection function: Concatenates multiple input columns together into a single column. + The function works with strings, numeric, binary and compatible array columns. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric or interval. + cols : :class:`~pyspark.sql.Column` or str + target column or columns to work on. + Each a column that evaluates to a string, numeric, binary, or array. - Examples - -------- - Example 1: Calculating the average age + Returns + ------- + :class:`~pyspark.sql.Column` + concatenated values. Type of the `Column` depends on input columns' type. + Returns a column of the same type as the input. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) - >>> df.select(sf.try_avg("age")).show() - +------------+ - |try_avg(age)| - +------------+ - | 8.5| - +------------+ + See Also + -------- + :meth:`pyspark.sql.functions.concat_ws` + :meth:`pyspark.sql.functions.array_join` : to concatenate string columns with delimiter - Example 2: Calculating the average age with None + Examples + -------- + Example 1: Concatenating string columns - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.try_avg("age")).show() + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([('abcd','123')], ['s', 'd']) + >>> df.select(sf.concat(df.s, df.d)).show() +------------+ - |try_avg(age)| + |concat(s, d)| +------------+ - | 3.0| + | abcd123| +------------+ - Example 3: Overflow results in NULL when ANSI mode is on + Example 2: Concatenating array columns - >>> from decimal import Decimal - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.createDataFrame( - ... [(Decimal("1" * 38),), (Decimal(0),)], "number DECIMAL(38, 0)") - ... df.select(sf.try_avg(df.number)).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2], [3, 4], [5]), ([1, 2], None, [3])], ['a', 'b', 'c']) + >>> df.select(sf.concat(df.a, df.b, df.c)).show() +---------------+ - |try_avg(number)| + |concat(a, b, c)| +---------------+ + |[1, 2, 3, 4, 5]| | NULL| +---------------+ - """ - return _invoke_function_over_columns("try_avg", col) - - -@_try_remote_functions -def try_sum(col: "ColumnOrName") -> Column: - """ - Returns the sum calculated from values of a group and the result is null on overflow. - .. versionadded:: 3.5.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to a numeric or interval. - - Examples - -------- - Example 1: Calculating the sum of values in a column + Example 3: Concatenating numeric columns >>> from pyspark.sql import functions as sf - >>> spark.range(10).select(sf.try_sum("id")).show() - +-----------+ - |try_sum(id)| - +-----------+ - | 45| - +-----------+ + >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) + >>> df.select(sf.concat(df.a, df.b, df.c)).show() + +---------------+ + |concat(a, b, c)| + +---------------+ + | 123| + +---------------+ - Example 2: Using a plus expression together to calculate the sum + Example 4: Concatenating binary columns >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) - >>> df.select(sf.try_sum(sf.col("A") + sf.col("B"))).show() - +----------------+ - |try_sum((A + B))| - +----------------+ - | 10| - +----------------+ - - Example 3: Calculating the summation of ages with None - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.try_sum("age")).show() - +------------+ - |try_sum(age)| - +------------+ - | 6| - +------------+ + >>> df = spark.createDataFrame([(bytearray(b'abc'), bytearray(b'def'))], ['a', 'b']) + >>> df.select(sf.concat(df.a, df.b)).show() + +-------------------+ + | concat(a, b)| + +-------------------+ + |[61 62 63 64 65 66]| + +-------------------+ - Example 4: Overflow results in NULL when ANSI mode is on + Example 5: Concatenating mixed types of columns - >>> from decimal import Decimal - >>> import pyspark.sql.functions as sf - >>> origin = spark.conf.get("spark.sql.ansi.enabled") - >>> spark.conf.set("spark.sql.ansi.enabled", "true") - >>> try: - ... df = spark.createDataFrame([(Decimal("1" * 38),)] * 10, "number DECIMAL(38, 0)") - ... df.select(sf.try_sum(df.number)).show() - ... finally: - ... spark.conf.set("spark.sql.ansi.enabled", origin) - +---------------+ - |try_sum(number)| - +---------------+ - | NULL| - +---------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,"abc",3,"def")], ['a','b','c','d']) + >>> df.select(sf.concat(df.a, df.b, df.c, df.d)).show() + +------------------+ + |concat(a, b, c, d)| + +------------------+ + | 1abc3def| + +------------------+ """ - return _invoke_function_over_columns("try_sum", col) + return _invoke_function_over_seq_of_columns("concat", cols) @_try_remote_functions -def mode(col: "ColumnOrName", deterministic: bool = False) -> Column: +def array_position(col: "ColumnOrName", value: Any) -> Column: """ - Returns the most frequent value in a group. + Array function: Locates the position of the first occurrence of the given value + in the given array. Returns null if either of the arguments are null. - .. versionadded:: 3.4.0 + .. versionadded:: 2.4.0 - .. versionchanged:: 4.0.0 - Supports deterministic argument. + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Notes + ----- + The position is not zero based, but 1 based index. Returns 0 if the given + value could not be found in the array. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column of any type. - deterministic : bool, optional - if there are multiple equally-frequent results then return the lowest (defaults to false). - A column that evaluates to a boolean. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + target column to work on. + A column that evaluates to an array. + value : Any + value or a :class:`~pyspark.sql.Column` expression to look for. + A column of the same type as the array elements. + + .. versionchanged:: 4.0.0 + `value` now also accepts a Column type. Returns ------- :class:`~pyspark.sql.Column` - the most frequent value in a group. + position of the value in the given array if found and 0 otherwise. + Returns a column that evaluates to a long. - Notes - ----- - Supports Spark Connect. + See Also + -------- + :meth:`pyspark.sql.functions.array_contains` Examples -------- + Example 1: Finding the position of a string in an array of strings + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.mode("year")).sort("course").show() - +------+----------+ - |course|mode(year)| - +------+----------+ - | Java| 2012| - |dotNET| 2012| - +------+----------+ + >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) + >>> df.select(sf.array_position(df.data, "a")).show() + +-----------------------+ + |array_position(data, a)| + +-----------------------+ + | 3| + +-----------------------+ - When multiple values have the same greatest frequency then either any of values is returned if - deterministic is false or is not defined, or the lowest value is returned if deterministic is - true. + Example 2: Finding the position of a string in an empty array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(-10,), (0,), (10,)], ["col"]) - >>> df.select(sf.mode("col", False)).show() # doctest: +SKIP - +---------+ - |mode(col)| - +---------+ - | 0| - +---------+ + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_position(df.data, "a")).show() + +-----------------------+ + |array_position(data, a)| + +-----------------------+ + | 0| + +-----------------------+ - >>> df.select(sf.mode("col", True)).show() - +---------------------------------------+ - |mode() WITHIN GROUP (ORDER BY col DESC)| - +---------------------------------------+ - | -10| - +---------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + Example 3: Finding the position of an integer in an array of integers - return _invoke_function("mode", _to_java_column(col), _enum_to_value(deterministic)) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_position(df.data, 2)).show() + +-----------------------+ + |array_position(data, 2)| + +-----------------------+ + | 2| + +-----------------------+ + + Example 4: Finding the position of a non-existing value in an array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["c", "b", "a"],)], ['data']) + >>> df.select(sf.array_position(df.data, "d")).show() + +-----------------------+ + |array_position(data, d)| + +-----------------------+ + | 0| + +-----------------------+ + + Example 5: Finding the position of a value in an array with nulls + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([None, "b", "a"],)], ['data']) + >>> df.select(sf.array_position(df.data, "a")).show() + +-----------------------+ + |array_position(data, a)| + +-----------------------+ + | 3| + +-----------------------+ + + Example 6: Finding the position of a column's value in an array of integers + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([10, 20, 30], 20)], ['data', 'col']) + >>> df.select(sf.array_position(df.data, df.col)).show() + +-------------------------+ + |array_position(data, col)| + +-------------------------+ + | 2| + +-------------------------+ + + """ + return _invoke_function_over_columns("array_position", col, lit(value)) @_try_remote_functions -def max(col: "ColumnOrName") -> Column: +def element_at(col: "ColumnOrName", extraction: Any) -> Column: """ - Aggregate function: returns the maximum value of the expression in a group. + Collection function: + (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will + throw an error. If index < 0, accesses elements from the last to the first. + If 'spark.sql.ansi.enabled' is set to true, an exception will be thrown if the index is out + of array boundaries instead of returning NULL. - .. versionadded:: 1.3.0 + (map, key) - Returns value for given key in `extraction` if col is map. The function always + returns NULL if the key is not contained in the map. + + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the maximum value is computed. + col : :class:`~pyspark.sql.Column` or str + name of column containing array or map. + A column that evaluates to an array or map. + extraction : + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. Returns ------- :class:`~pyspark.sql.Column` - A column that contains the maximum value computed. - - See Also - -------- - :meth:`pyspark.sql.functions.min` - :meth:`pyspark.sql.functions.avg` - :meth:`pyspark.sql.functions.sum` + value at given position. + Returns a column of the element type of the input array, or the value type of the input map. Notes ----- - - Null values are ignored during the computation. - - NaN values are larger than any other numeric value. + The position is not zero based, but 1 based index. + If extraction is a string, :meth:`element_at` treats it as a literal string, + while :meth:`try_element_at` treats it as a column name. - Examples + See Also -------- - Example 1: Compute the maximum value of a numeric column - - >>> import pyspark.sql.functions as sf - >>> df = spark.range(10) - >>> df.select(sf.max(df.id)).show() - +-------+ - |max(id)| - +-------+ - | 9| - +-------+ + :meth:`pyspark.sql.functions.get` + :meth:`pyspark.sql.functions.try_element_at` - Example 2: Compute the maximum value of a string column + Examples + -------- + Example 1: Getting the first element of an array - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("A",), ("B",), ("C",)], ["value"]) - >>> df.select(sf.max(df.value)).show() - +----------+ - |max(value)| - +----------+ - | C| - +----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.element_at(df.data, 1)).show() + +-------------------+ + |element_at(data, 1)| + +-------------------+ + | a| + +-------------------+ - Example 3: Compute the maximum value of a column in a grouped DataFrame + Example 2: Getting the last element of an array using negative index - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("A", 1), ("A", 2), ("B", 3), ("B", 4)], ["key", "value"]) - >>> df.groupBy("key").agg(sf.max(df.value)).show() - +---+----------+ - |key|max(value)| - +---+----------+ - | A| 2| - | B| 4| - +---+----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.element_at(df.data, -1)).show() + +--------------------+ + |element_at(data, -1)| + +--------------------+ + | c| + +--------------------+ - Example 4: Compute the maximum value of multiple columns in a grouped DataFrame + Example 3: Getting a value from a map using a key - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame( - ... [("A", 1, 2), ("A", 2, 3), ("B", 3, 4), ("B", 4, 5)], ["key", "value1", "value2"]) - >>> df.groupBy("key").agg(sf.max("value1"), sf.max("value2")).show() - +---+-----------+-----------+ - |key|max(value1)|max(value2)| - +---+-----------+-----------+ - | A| 2| 3| - | B| 4| 5| - +---+-----------+-----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.element_at(df.data, sf.lit("a"))).show() + +-------------------+ + |element_at(data, a)| + +-------------------+ + | 1.0| + +-------------------+ - Example 5: Compute the maximum value of a column with null values + Example 4: Getting a non-existing value from a map using a key - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (None,)], ["value"]) - >>> df.select(sf.max(df.value)).show() - +----------+ - |max(value)| - +----------+ - | 2| - +----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.element_at(df.data, sf.lit("c"))).show() + +-------------------+ + |element_at(data, c)| + +-------------------+ + | NULL| + +-------------------+ - Example 6: Compute the maximum value of a column with "NaN" values + Example 5: Getting a value from a map using a literal string as the key - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1.1,), (float("nan"),), (3.3,)], ["value"]) - >>> df.select(sf.max(df.value)).show() - +----------+ - |max(value)| - +----------+ - | NaN| - +----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) + >>> df.select(sf.element_at(df.data, 'b')).show() + +-------------------+ + |element_at(data, b)| + +-------------------+ + | 2.0| + +-------------------+ """ - return _invoke_function_over_columns("max", col) + return _invoke_function_over_columns("element_at", col, lit(extraction)) @_try_remote_functions -def min(col: "ColumnOrName") -> Column: +def try_element_at(col: "ColumnOrName", extraction: "ColumnOrName") -> Column: """ - Aggregate function: returns the minimum value of the expression in a group. + Collection function: + (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will + throw an error. If index < 0, accesses elements from the last to the first. The function + always returns NULL if the index exceeds the length of the array. - .. versionadded:: 1.3.0 + (map, key) - Returns value for given key. The function always returns NULL if the key is not + contained in the map. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the minimum value is computed. + col : :class:`~pyspark.sql.Column` or str + name of column containing array or map. + A column that evaluates to an array or map. + extraction : + index to check for in array or key to check for in map. + A column that evaluates to an integer for an array, or the key type for a map. Returns ------- :class:`~pyspark.sql.Column` - A column that contains the minimum value computed. + Returns a column of the element type of the input array, or the value type of the input map. + + Notes + ----- + The position is not zero based, but 1 based index. + If extraction is a string, :meth:`try_element_at` treats it as a column name, + while :meth:`element_at` treats it as a literal string. See Also -------- - :meth:`pyspark.sql.functions.max` - :meth:`pyspark.sql.functions.avg` - :meth:`pyspark.sql.functions.sum` + :meth:`pyspark.sql.functions.get` + :meth:`pyspark.sql.functions.element_at` Examples -------- - Example 1: Compute the minimum value of a numeric column + Example 1: Getting the first element of an array - >>> import pyspark.sql.functions as sf - >>> df = spark.range(10) - >>> df.select(sf.min(df.id)).show() - +-------+ - |min(id)| - +-------+ - | 0| - +-------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit(1))).show() + +-----------------------+ + |try_element_at(data, 1)| + +-----------------------+ + | a| + +-----------------------+ - Example 2: Compute the minimum value of a string column + Example 2: Getting the last element of an array using negative index - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Alice",), ("Bob",), ("Charlie",)], ["name"]) - >>> df.select(sf.min("name")).show() - +---------+ - |min(name)| - +---------+ - | Alice| - +---------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit(-1))).show() + +------------------------+ + |try_element_at(data, -1)| + +------------------------+ + | c| + +------------------------+ - Example 3: Compute the minimum value of a column with null values + Example 3: Getting a value from a map using a key - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1,), (None,), (3,)], ["value"]) - >>> df.select(sf.min("value")).show() - +----------+ - |min(value)| - +----------+ - | 1| - +----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit("a"))).show() + +-----------------------+ + |try_element_at(data, a)| + +-----------------------+ + | 1.0| + +-----------------------+ - Example 4: Compute the minimum value of a column in a grouped DataFrame + Example 4: Getting a non-existing element from an array - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("Alice", 1), ("Alice", 2), ("Bob", 3)], ["name", "value"]) - >>> df.groupBy("name").agg(sf.min("value")).show() - +-----+----------+ - | name|min(value)| - +-----+----------+ - |Alice| 1| - | Bob| 3| - +-----+----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit(4))).show() + +-----------------------+ + |try_element_at(data, 4)| + +-----------------------+ + | NULL| + +-----------------------+ - Example 5: Compute the minimum value of a column in a DataFrame with multiple columns + Example 5: Getting a non-existing value from a map using a key - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame( - ... [("Alice", 1, 100), ("Bob", 2, 200), ("Charlie", 3, 300)], - ... ["name", "value1", "value2"]) - >>> df.select(sf.min("value1"), sf.min("value2")).show() - +-----------+-----------+ - |min(value1)|min(value2)| - +-----------+-----------+ - | 1| 100| - +-----------+-----------+ - """ - return _invoke_function_over_columns("min", col) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0},)], ['data']) + >>> df.select(sf.try_element_at(df.data, sf.lit("c"))).show() + +-----------------------+ + |try_element_at(data, c)| + +-----------------------+ + | NULL| + +-----------------------+ + Example 6: Getting a value from a map using a column name as the key -@_try_remote_functions -def max_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([({"a": 1.0, "b": 2.0}, "a")], ['data', 'b']) + >>> df.select(sf.try_element_at(df.data, 'b')).show() + +-----------------------+ + |try_element_at(data, b)| + +-----------------------+ + | 1.0| + +-----------------------+ """ - Returns the value(s) from the `col` parameter that are associated with the maximum value(s) - from the `ord` parameter. This function is often used to find the `col` parameter value - corresponding to the maximum `ord` parameter value within each group when used with groupBy(). - - When `k` is specified, returns an array of up to `k` values associated with the top `k` - maximum values from `ord`. - - .. versionadded:: 3.3.0 + return _invoke_function_over_columns("try_element_at", col, extraction) - .. versionchanged:: 3.4.0 - Supports Spark Connect. - .. versionchanged:: 4.2.0 - Added optional `k` parameter to return top-k values. +@_try_remote_functions +def get(col: "ColumnOrName", index: Union["ColumnOrName", int]) -> Column: + """ + Array function: Returns the element of an array at the given (0-based) index. + If the index points outside of the array boundaries, then this function + returns NULL. - Notes - ----- - The function is non-deterministic so the output order can be different for those - associated the same values of `col`. + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column representing the values to be returned. This could be the column instance - or the column name as string. - A column of any type. - ord : :class:`~pyspark.sql.Column` or column name - The column that needs to be maximized. This could be the column instance - or the column name as string. - A column of any orderable type. - k : int, optional - If specified, returns an array of up to `k` values associated with the top `k` - maximum ordering values, sorted in descending order by the ordering column. - Must be a positive integer literal <= 100000. - A column that evaluates to an integer. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + Name of the column containing the array. + A column that evaluates to an array. + index : :class:`~pyspark.sql.Column` or str or int + Index to check for in the array. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A column object representing the value from `col` that is associated with - the maximum value from `ord`. If `k` is specified, returns an array of values. + Value at the given position. + Returns a column of the element type of the input array. + + Notes + ----- + The position is not 1-based, but 0-based index. + Supports Spark Connect. + + See Also + -------- + :meth:`pyspark.sql.functions.element_at` + :meth:`pyspark.sql.functions.try_element_at` Examples -------- - Example 1: Using `max_by` with groupBy + Example 1: Getting an element at a fixed position - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.max_by("year", "earnings")).sort("course").show() - +------+----------------------+ - |course|max_by(year, earnings)| - +------+----------------------+ - | Java| 2013| - |dotNET| 2013| - +------+----------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.get(df.data, 1)).show() + +------------+ + |get(data, 1)| + +------------+ + | b| + +------------+ - Example 2: Using `max_by` with different data types + Example 2: Getting an element at a position outside the array boundaries - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), - ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.max_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|max_by(name, years_in_dept)| - +----------+---------------------------+ - | IT| Charlie| - | Marketing| Anna| - +----------+---------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"],)], ['data']) + >>> df.select(sf.get(df.data, 3)).show() + +------------+ + |get(data, 3)| + +------------+ + | NULL| + +------------+ - Example 3: Using `max_by` where `ord` has multiple maximum values + Example 3: Getting an element at a position specified by another column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), - ... ("Finance", "George", 9), ("Consult", "Henry", 7)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.max_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|max_by(name, years_in_dept)| - +----------+---------------------------+ - | Consult| Henry| - | Finance| George| - +----------+---------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) + >>> df.select(sf.get(df.data, df.index)).show() + +----------------+ + |get(data, index)| + +----------------+ + | c| + +----------------+ - Example 4: Using `max_by` with `k` to get top-k values - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], - ... schema=("x", "y")) - >>> df.select(sf.max_by("x", "y", 2)).show() - +---------------+ - |max_by(x, y, 2)| - +---------------+ - | [b, d]| - +---------------+ - """ - if k is not None: - return _invoke_function_over_columns("max_by", col, ord, lit(k)) - return _invoke_function_over_columns("max_by", col, ord) + Example 4: Getting an element at a position calculated from another column + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"], 2)], ['data', 'index']) + >>> df.select(sf.get(df.data, df.index - 1)).show() + +----------------------+ + |get(data, (index - 1))| + +----------------------+ + | b| + +----------------------+ -@_try_remote_functions -def min_by(col: "ColumnOrName", ord: "ColumnOrName", k: Optional[int] = None) -> Column: - """ - Returns the value(s) from the `col` parameter that are associated with the minimum value(s) - from the `ord` parameter. This function is often used to find the `col` parameter value - corresponding to the minimum `ord` parameter value within each group when used with groupBy(). + Example 5: Getting an element at a negative position - When `k` is specified, returns an array of up to `k` values associated with the bottom `k` - minimum values from `ord`. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(["a", "b", "c"], )], ['data']) + >>> df.select(sf.get(df.data, -1)).show() + +-------------+ + |get(data, -1)| + +-------------+ + | NULL| + +-------------+ + """ + index = _enum_to_value(index) + index = lit(index) if isinstance(index, int) else index - .. versionadded:: 3.3.0 + return _invoke_function_over_columns("get", col, index) - .. versionchanged:: 3.4.0 - Supports Spark Connect. - .. versionchanged:: 4.2.0 - Added optional `k` parameter to return bottom-k values. +@_try_remote_functions +def array_prepend(col: "ColumnOrName", value: Any) -> Column: + """ + Array function: Returns an array containing the given element as + the first element and the rest of the elements from the original array. - Notes - ----- - The function is non-deterministic so the output order can be different for those - associated the same values of `col`. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column representing the values that will be returned. This could be the column instance - or the column name as string. - A column of any type. - ord : :class:`~pyspark.sql.Column` or column name - The column that needs to be minimized. This could be the column instance - or the column name as string. - A column of any orderable type. - k : int, optional - If specified, returns an array of up to `k` values associated with the bottom `k` - minimum ordering values, sorted in ascending order by the ordering column. - Must be a positive integer literal <= 100000. - A column that evaluates to an integer. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + name of column containing array. + A column that evaluates to an array. + value : + a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` - Column object that represents the value from `col` associated with - the minimum value from `ord`. If `k` is specified, returns an array of values. + an array with the given value prepended. + Returns a column that evaluates to an array. + + See Also + -------- + :meth:`pyspark.sql.functions.array_append` + :meth:`pyspark.sql.functions.array_insert` Examples -------- - Example 1: Using `min_by` with groupBy: + Example 1: Prepending a column value to an array column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.min_by("year", "earnings")).sort("course").show() - +------+----------------------+ - |course|min_by(year, earnings)| - +------+----------------------+ - | Java| 2012| - |dotNET| 2012| - +------+----------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) + >>> df.select(sf.array_prepend(df.c1, df.c2)).show() + +---------------------+ + |array_prepend(c1, c2)| + +---------------------+ + | [c, b, a, c]| + +---------------------+ - Example 2: Using `min_by` with different data types: + Example 2: Prepending a numeric value to an array column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Marketing", "Anna", 4), ("IT", "Bob", 2), - ... ("IT", "Charlie", 3), ("Marketing", "David", 1)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.min_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|min_by(name, years_in_dept)| - +----------+---------------------------+ - | IT| Bob| - | Marketing| David| - +----------+---------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_prepend(df.data, 4)).show() + +----------------------+ + |array_prepend(data, 4)| + +----------------------+ + | [4, 1, 2, 3]| + +----------------------+ - Example 3: Using `min_by` where `ord` has multiple minimum values: + Example 3: Prepending a null value to an array column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("Consult", "Eva", 6), ("Finance", "Frank", 5), - ... ("Finance", "George", 9), ("Consult", "Henry", 7)], - ... schema=("department", "name", "years_in_dept")) - >>> df.groupby("department").agg( - ... sf.min_by("name", "years_in_dept") - ... ).sort("department").show() - +----------+---------------------------+ - |department|min_by(name, years_in_dept)| - +----------+---------------------------+ - | Consult| Eva| - | Finance| Frank| - +----------+---------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_prepend(df.data, None)).show() + +-------------------------+ + |array_prepend(data, NULL)| + +-------------------------+ + | [NULL, 1, 2, 3]| + +-------------------------+ - Example 4: Using `min_by` with `k` to get bottom-k values + Example 4: Prepending a value to a NULL array column - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([ - ... ("a", 10), ("b", 50), ("c", 20), ("d", 40)], - ... schema=("x", "y")) - >>> df.select(sf.min_by("x", "y", 2)).show() - +---------------+ - |min_by(x, y, 2)| - +---------------+ - | [a, c]| - +---------------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([(None,)], schema=schema) + >>> df.select(sf.array_prepend(df.data, 4)).show() + +----------------------+ + |array_prepend(data, 4)| + +----------------------+ + | NULL| + +----------------------+ + + Example 5: Prepending a value to an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_prepend(df.data, 1)).show() + +----------------------+ + |array_prepend(data, 1)| + +----------------------+ + | [1]| + +----------------------+ """ - if k is not None: - return _invoke_function_over_columns("min_by", col, ord, lit(k)) - return _invoke_function_over_columns("min_by", col, ord) + return _invoke_function_over_columns("array_prepend", col, lit(value)) @_try_remote_functions -def count(col: "ColumnOrName") -> Column: +def array_remove(col: "ColumnOrName", element: Any) -> Column: """ - Aggregate function: returns the number of items in a group. + Array function: Remove all elements that equal to element from the given array. - .. versionadded:: 1.3.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + col : :class:`~pyspark.sql.Column` or str + name of column containing array. + A column that evaluates to an array. + element : + element or a :class:`~pyspark.sql.Column` expression to be removed from the array. + A column of the same type as the array elements. + + .. versionchanged:: 4.0.0 + `element` now also accepts a Column type. Returns ------- :class:`~pyspark.sql.Column` - column for computed results. + A new column that is an array excluding the given value from the input column. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.count_if` + :meth:`pyspark.sql.functions.array_compact` Examples -------- - Example 1: Count all rows in a DataFrame + Example 1: Removing a specific value from a simple array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), ("a",), ("b",), ("c",)], schema=["alphabets"]) - >>> df.select(sf.count(sf.expr("*"))).show() - +--------+ - |count(1)| - +--------+ - | 4| - +--------+ + >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],)], ['data']) + >>> df.select(sf.array_remove(df.data, 1)).show() + +---------------------+ + |array_remove(data, 1)| + +---------------------+ + | [2, 3]| + +---------------------+ - Example 2: Count non-null values in a specific column + Example 2: Removing a specific value from multiple arrays >>> from pyspark.sql import functions as sf - >>> df.select(sf.count(df.alphabets)).show() - +----------------+ - |count(alphabets)| - +----------------+ - | 3| - +----------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 1, 1],), ([4, 5, 5, 4],)], ['data']) + >>> df.select(sf.array_remove(df.data, 5)).show() + +---------------------+ + |array_remove(data, 5)| + +---------------------+ + | [1, 2, 3, 1, 1]| + | [4, 4]| + +---------------------+ - Example 3: Count all rows in a DataFrame with multiple columns + Example 3: Removing a value that does not exist in the array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, "apple"), (2, "banana"), (3, None)], schema=["id", "fruit"]) - >>> df.select(sf.count(sf.expr("*"))).show() - +--------+ - |count(1)| - +--------+ - | 3| - +--------+ + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_remove(df.data, 4)).show() + +---------------------+ + |array_remove(data, 4)| + +---------------------+ + | [1, 2, 3]| + +---------------------+ - Example 4: Count non-null values in multiple columns + Example 4: Removing a value from an array with all identical values >>> from pyspark.sql import functions as sf - >>> df.select(sf.count(df.id), sf.count(df.fruit)).show() - +---------+------------+ - |count(id)|count(fruit)| - +---------+------------+ - | 3| 2| - +---------+------------+ + >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) + >>> df.select(sf.array_remove(df.data, 1)).show() + +---------------------+ + |array_remove(data, 1)| + +---------------------+ + | []| + +---------------------+ + + Example 5: Removing a value from an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema) + >>> df.select(sf.array_remove(df.data, 1)).show() + +---------------------+ + |array_remove(data, 1)| + +---------------------+ + | []| + +---------------------+ + + Example 6: Removing a column's value from a simple array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3, 1, 1], 1)], ['data', 'col']) + >>> df.select(sf.array_remove(df.data, df.col)).show() + +-----------------------+ + |array_remove(data, col)| + +-----------------------+ + | [2, 3]| + +-----------------------+ """ - return _invoke_function_over_columns("count", col) + return _invoke_function_over_columns("array_remove", col, lit(element)) @_try_remote_functions -def sum(col: "ColumnOrName") -> Column: +def array_distinct(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the sum of all values in the expression. + Array function: removes duplicate values from the array. - .. versionadded:: 1.3.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric or interval. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + A new column that is an array of unique values from the input column. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.min` - :meth:`pyspark.sql.functions.max` - :meth:`pyspark.sql.functions.avg` + :meth:`pyspark.sql.functions.array_except` + :meth:`pyspark.sql.functions.array_intersect` + :meth:`pyspark.sql.functions.array_union` Examples -------- - Example 1: Calculating the sum of values in a column + Example 1: Removing duplicate values from a simple array >>> from pyspark.sql import functions as sf - >>> df = spark.range(10) - >>> df.select(sf.sum(df["id"])).show() - +-------+ - |sum(id)| - +-------+ - | 45| - +-------+ + >>> df = spark.createDataFrame([([1, 2, 3, 2],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1, 2, 3]| + +--------------------+ - Example 2: Using a plus expression together to calculate the sum + Example 2: Removing duplicate values from multiple arrays >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2), (3, 4)], ["A", "B"]) - >>> df.select(sf.sum(sf.col("A") + sf.col("B"))).show() - +------------+ - |sum((A + B))| - +------------+ - | 10| - +------------+ + >>> df = spark.createDataFrame([([1, 2, 3, 2],), ([4, 5, 5, 4],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1, 2, 3]| + | [4, 5]| + +--------------------+ - Example 3: Calculating the summation of ages with None + Example 3: Removing duplicate values from an array with all identical values - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.sum("age")).show() - +--------+ - |sum(age)| - +--------+ - | 6| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 1, 1],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1]| + +--------------------+ + + Example 4: Removing duplicate values from an array with no duplicate values + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | [1, 2, 3]| + +--------------------+ + + Example 5: Removing duplicate values from an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema) + >>> df.select(sf.array_distinct(df.data)).show() + +--------------------+ + |array_distinct(data)| + +--------------------+ + | []| + +--------------------+ """ - return _invoke_function_over_columns("sum", col) + return _invoke_function_over_columns("array_distinct", col) @_try_remote_functions -def avg(col: "ColumnOrName") -> Column: +def array_insert(arr: "ColumnOrName", pos: Union["ColumnOrName", int], value: Any) -> Column: """ - Aggregate function: returns the average of the values in a group. - - .. versionadded:: 1.3.0 + Array function: Inserts an item into a given array at a specified array index. + Array indices start at 1, or start from the end if index is negative. + Index above array size appends the array, or prepends the array if index is negative, + with 'null' elements. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric or interval. + arr : :class:`~pyspark.sql.Column` or str + name of column containing an array. + A column that evaluates to an array. + pos : :class:`~pyspark.sql.Column` or str or int + name of integral type column indicating position of insertion + (starting at index 1, negative position is a start from the back of the array). + A column that evaluates to an integer. + value : + a literal value, or a :class:`~pyspark.sql.Column` expression. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + an array of values, including the new specified value + Returns a column that evaluates to an array. + + Notes + ----- + Supports Spark Connect. See Also -------- - :meth:`pyspark.sql.functions.min` - :meth:`pyspark.sql.functions.max` - :meth:`pyspark.sql.functions.sum` + :meth:`pyspark.sql.functions.array_append` + :meth:`pyspark.sql.functions.array_prepend` Examples -------- - Example 1: Calculating the average age + Example 1: Inserting a value at a specific position - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) - >>> df.select(sf.avg("age")).show() - +--------+ - |avg(age)| - +--------+ - | 8.5| - +--------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, 2, 'd')).show() + +------------------------+ + |array_insert(data, 2, d)| + +------------------------+ + | [a, d, b, c]| + +------------------------+ - Example 2: Calculating the average age with None + Example 2: Inserting a value at a negative position - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.avg("age")).show() - +--------+ - |avg(age)| - +--------+ - | 3.0| - +--------+ - """ - return _invoke_function_over_columns("avg", col) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, -2, 'd')).show() + +-------------------------+ + |array_insert(data, -2, d)| + +-------------------------+ + | [a, b, d, c]| + +-------------------------+ + Example 3: Inserting a value at a position greater than the array size -@_try_remote_functions -def mean(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the average of the values in a group. - An alias of :func:`avg`. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, 5, 'e')).show() + +------------------------+ + |array_insert(data, 5, e)| + +------------------------+ + | [a, b, c, NULL, e]| + +------------------------+ - .. versionadded:: 1.4.0 + Example 4: Inserting a NULL value - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['a', 'b', 'c'],)], ['data']) + >>> df.select(sf.array_insert(df.data, 2, sf.lit(None))).show() + +---------------------------+ + |array_insert(data, 2, NULL)| + +---------------------------+ + | [a, NULL, b, c]| + +---------------------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric or interval. - - Returns - ------- - :class:`~pyspark.sql.Column` - the column for computed results. - - Examples - -------- - Example 1: Calculating the average age - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, 15), (1990, 2)], ["birth", "age"]) - >>> df.select(sf.mean("age")).show() - +--------+ - |avg(age)| - +--------+ - | 8.5| - +--------+ - - Example 2: Calculating the average age with None + Example 5: Inserting a value into a NULL array - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1982, None), (1990, 2), (2000, 4)], ["birth", "age"]) - >>> df.select(sf.mean("age")).show() - +--------+ - |avg(age)| - +--------+ - | 3.0| - +--------+ + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) + >>> df = spark.createDataFrame([(None,)], schema=schema) + >>> df.select(sf.array_insert(df.data, 1, 5)).show() + +------------------------+ + |array_insert(data, 1, 5)| + +------------------------+ + | NULL| + +------------------------+ """ - return _invoke_function_over_columns("mean", col) + pos = _enum_to_value(pos) + pos = lit(pos) if isinstance(pos, int) else pos + + return _invoke_function_over_columns("array_insert", arr, pos, lit(value)) @_try_remote_functions -def median(col: "ColumnOrName") -> Column: +def array_intersect(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns the median of the values in a group. + Array function: returns a new array containing the intersection of elements in col1 and col2, + without duplicates. - .. versionadded:: 3.4.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric, interval, or time. + col1 : :class:`~pyspark.sql.Column` or str + Name of column containing the first array. + A column that evaluates to an array. + col2 : :class:`~pyspark.sql.Column` or str + Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - the median of the values in a group. + A new array containing the intersection of elements in col1 and col2. + Returns a column that evaluates to an array. Notes ----- - Supports Spark Connect. - - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.approx_percentile` - :meth:`pyspark.sql.functions.percentile_approx` + This function does not preserve the order of the elements in the input arrays. See Also -------- - :meth:`pyspark.sql.functions.approx_percentile` - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.percentile_approx` + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.array_except` + :meth:`pyspark.sql.functions.array_union` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([ - ... ("Java", 2012, 20000), ("dotNET", 2012, 5000), - ... ("Java", 2012, 22000), ("dotNET", 2012, 10000), - ... ("dotNET", 2013, 48000), ("Java", 2013, 30000)], - ... schema=("course", "year", "earnings")) - >>> df.groupby("course").agg(sf.median("earnings")).show() - +------+----------------+ - |course|median(earnings)| - +------+----------------+ - | Java| 22000.0| - |dotNET| 10000.0| - +------+----------------+ - """ - return _invoke_function_over_columns("median", col) + Example 1: Basic usage + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) + >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() + +-----------------------------------------+ + |sort_array(array_intersect(c1, c2), true)| + +-----------------------------------------+ + | [a, c]| + +-----------------------------------------+ -@_try_remote_functions -def sumDistinct(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the sum of distinct values in the expression. + Example 2: Intersection with no common elements - .. versionadded:: 1.3.0 + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) + >>> df.select(sf.array_intersect(df.c1, df.c2)).show() + +-----------------------+ + |array_intersect(c1, c2)| + +-----------------------+ + | []| + +-----------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Intersection with all common elements - .. deprecated:: 3.2.0 - Use :func:`sum_distinct` instead. + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) + >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() + +-----------------------------------------+ + |sort_array(array_intersect(c1, c2), true)| + +-----------------------------------------+ + | [a, b, c]| + +-----------------------------------------+ + + Example 4: Intersection with null values + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) + >>> df.select(sf.sort_array(sf.array_intersect(df.c1, df.c2))).show() + +-----------------------------------------+ + |sort_array(array_intersect(c1, c2), true)| + +-----------------------------------------+ + | [NULL, a]| + +-----------------------------------------+ + + Example 5: Intersection with empty arrays + + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> data = [Row(c1=[], c2=["a", "b", "c"])] + >>> schema = StructType([ + ... StructField("c1", ArrayType(StringType()), True), + ... StructField("c2", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.array_intersect(df.c1, df.c2)).show() + +-----------------------+ + |array_intersect(c1, c2)| + +-----------------------+ + | []| + +-----------------------+ """ - warnings.warn("Deprecated in 3.2, use sum_distinct instead.", FutureWarning) - return sum_distinct(col) + return _invoke_function_over_columns("array_intersect", col1, col2) @_try_remote_functions -def sum_distinct(col: "ColumnOrName") -> Column: +def array_union(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Aggregate function: returns the sum of distinct values in the expression. + Array function: returns a new array containing the union of elements in col1 and col2, + without duplicates. - .. versionadded:: 3.2.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. + col1 : :class:`~pyspark.sql.Column` or str + Name of column containing the first array. + A column that evaluates to an array. + col2 : :class:`~pyspark.sql.Column` or str + Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + A new array containing the union of elements in col1 and col2. + Returns a column that evaluates to an array. + + Notes + ----- + This function does not preserve the order of the elements in the input arrays. + + See Also + -------- + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.array_except` + :meth:`pyspark.sql.functions.array_intersect` Examples -------- - Example 1: Using sum_distinct function on a column with all distinct values + Example 1: Basic usage - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,)], ["numbers"]) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | 10| - +---------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c, d, f]| + +-------------------------------------+ - Example 2: Using sum_distinct function on a column with no distinct values + Example 2: Union with no common elements - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (1,), (1,), (1,)], ["numbers"]) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | 1| - +---------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c, d, e, f]| + +-------------------------------------+ - Example 3: Using sum_distinct function on a column with null and duplicate values + Example 3: Union with all common elements - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(None,), (1,), (1,), (2,)], ["numbers"]) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | 3| - +---------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c]| + +-------------------------------------+ - Example 4: Using sum_distinct function on a column with all None values + Example 4: Union with null values - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, IntegerType - >>> schema = StructType([StructField("numbers", IntegerType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.sum_distinct('numbers')).show() - +---------------------+ - |sum(DISTINCT numbers)| - +---------------------+ - | NULL| - +---------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [NULL, a, b, c]| + +-------------------------------------+ + + Example 5: Union with empty arrays + + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> data = [Row(c1=[], c2=["a", "b", "c"])] + >>> schema = StructType([ + ... StructField("c1", ArrayType(StringType()), True), + ... StructField("c2", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.sort_array(sf.array_union(df.c1, df.c2))).show() + +-------------------------------------+ + |sort_array(array_union(c1, c2), true)| + +-------------------------------------+ + | [a, b, c]| + +-------------------------------------+ """ - return _invoke_function_over_columns("sum_distinct", col) + return _invoke_function_over_columns("array_union", col1, col2) @_try_remote_functions -def listagg(col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None) -> Column: +def array_except(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Aggregate function: returns the concatenation of non-null input values, - separated by the delimiter. + Array function: returns a new array containing the elements present in col1 but not in col2, + without duplicates. - .. versionadded:: 4.0.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a string or binary. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. - A column that evaluates to a string, binary, or null. Must be a constant. + col1 : :class:`~pyspark.sql.Column` or str + Name of column containing the first array. + A column that evaluates to an array. + col2 : :class:`~pyspark.sql.Column` or str + Name of column containing the second array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + A new array containing the elements present in col1 but not in col2. + Returns a column that evaluates to an array. + + Notes + ----- + This function does not preserve the order of the elements in the input arrays. + + See Also + -------- + :meth:`pyspark.sql.functions.array_distinct` + :meth:`pyspark.sql.functions.array_intersect` + :meth:`pyspark.sql.functions.array_union` Examples -------- - Example 1: Using listagg function + Example 1: Basic usage - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.listagg('strings')).show() - +----------------------+ - |listagg(strings, NULL)| - +----------------------+ - | abc| - +----------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["c", "d", "a", "f"])]) + >>> df.select(sf.array_except(df.c1, df.c2)).show() + +--------------------+ + |array_except(c1, c2)| + +--------------------+ + | [b]| + +--------------------+ - Example 2: Using listagg function with a delimiter + Example 2: Except with no common elements - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.listagg('strings', ', ')).show() + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2=["d", "e", "f"])]) + >>> df.select(sf.sort_array(sf.array_except(df.c1, df.c2))).show() + +--------------------------------------+ + |sort_array(array_except(c1, c2), true)| + +--------------------------------------+ + | [a, b, c]| + +--------------------------------------+ + + Example 3: Except with all common elements + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", "c"], c2=["a", "b", "c"])]) + >>> df.select(sf.array_except(df.c1, df.c2)).show() +--------------------+ - |listagg(strings, , )| + |array_except(c1, c2)| +--------------------+ - | a, b, c| + | []| +--------------------+ - Example 3: Using listagg function with a binary column and delimiter + Example 4: Except with null values - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) - >>> df.select(sf.listagg('bytes', b'\x42')).show() - +---------------------+ - |listagg(bytes, X'42')| - +---------------------+ - | [01 42 02 42 03]| - +---------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["a", "b", None], c2=["a", None, "c"])]) + >>> df.select(sf.array_except(df.c1, df.c2)).show() + +--------------------+ + |array_except(c1, c2)| + +--------------------+ + | [b]| + +--------------------+ - Example 4: Using listagg function on a column with all None values + Example 5: Except with empty arrays - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.listagg('strings')).show() - +----------------------+ - |listagg(strings, NULL)| - +----------------------+ - | NULL| - +----------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> data = [Row(c1=[], c2=["a", "b", "c"])] + >>> schema = StructType([ + ... StructField("c1", ArrayType(StringType()), True), + ... StructField("c2", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.array_except(df.c1, df.c2)).show() + +--------------------+ + |array_except(c1, c2)| + +--------------------+ + | []| + +--------------------+ """ - if delimiter is None: - return _invoke_function_over_columns("listagg", col) - else: - return _invoke_function_over_columns("listagg", col, lit(delimiter)) + return _invoke_function_over_columns("array_except", col1, col2) @_try_remote_functions -def listagg_distinct( - col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None -) -> Column: +def array_compact(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the concatenation of distinct non-null input values, - separated by the delimiter. + Array function: removes null values from the array. - .. versionadded:: 4.0.0 + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + A new column that is an array excluding the null values from the input column. + Returns a column that evaluates to an array. + + Notes + ----- + Supports Spark Connect. + + See Also + -------- + :meth:`pyspark.sql.functions.array_remove` Examples -------- - Example 1: Using listagg_distinct function + Example 1: Removing null values from a simple array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.listagg_distinct('strings')).show() - +-------------------------------+ - |listagg(DISTINCT strings, NULL)| - +-------------------------------+ - | abc| - +-------------------------------+ + >>> df = spark.createDataFrame([([1, None, 2, 3],)], ['data']) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | [1, 2, 3]| + +-------------------+ - Example 2: Using listagg_distinct function with a delimiter + Example 2: Removing null values from multiple arrays >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.listagg_distinct('strings', ', ')).show() - +-----------------------------+ - |listagg(DISTINCT strings, , )| - +-----------------------------+ - | a, b, c| - +-----------------------------+ + >>> df = spark.createDataFrame([([1, None, 2, 3],), ([4, 5, None, 4],)], ['data']) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | [1, 2, 3]| + | [4, 5, 4]| + +-------------------+ - Example 3: Using listagg_distinct function with a binary column and delimiter + Example 3: Removing null values from an array with all null values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], - ... ['bytes']) - >>> df.select(sf.listagg_distinct('bytes', b'\x42')).show() - +------------------------------+ - |listagg(DISTINCT bytes, X'42')| - +------------------------------+ - | [01 42 02 42 03]| - +------------------------------+ + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([ + ... StructField("data", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame([([None, None, None],)], schema) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | []| + +-------------------+ - Example 4: Using listagg_distinct function on a column with all None values + Example 4: Removing null values from an array with no null values >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.listagg_distinct('strings')).show() - +-------------------------------+ - |listagg(DISTINCT strings, NULL)| - +-------------------------------+ - | NULL| - +-------------------------------+ - """ - if delimiter is None: - return _invoke_function_over_columns("listagg_distinct", col) - else: - return _invoke_function_over_columns("listagg_distinct", col, lit(delimiter)) - - -@_try_remote_functions -def string_agg( - col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None -) -> Column: + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | [1, 2, 3]| + +-------------------+ + + Example 5: Removing null values from an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([ + ... StructField("data", ArrayType(StringType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema) + >>> df.select(sf.array_compact(df.data)).show() + +-------------------+ + |array_compact(data)| + +-------------------+ + | []| + +-------------------+ """ - Aggregate function: returns the concatenation of non-null input values, - separated by the delimiter. + return _invoke_function_over_columns("array_compact", col) - An alias of :func:`listagg`. - .. versionadded:: 4.0.0 +@_try_remote_functions +def array_append(col: "ColumnOrName", value: Any) -> Column: + """ + Array function: returns a new array column by appending `value` to the existing array `col`. + + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a string or binary. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. - A column that evaluates to a string, binary, or null. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + The name of the column containing the array. + A column that evaluates to an array. + value : + A literal value, or a :class:`~pyspark.sql.Column` expression to be appended to the array. + A column of the same type as the array elements. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + A new array column with `value` appended to the original array. + Returns a column that evaluates to an array. + + Notes + ----- + Supports Spark Connect. + + See Also + -------- + :meth:`pyspark.sql.functions.array_insert` + :meth:`pyspark.sql.functions.array_prepend` Examples -------- - Example 1: Using string_agg function + Example 1: Appending a column value to an array column - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.string_agg('strings')).show() - +-------------------------+ - |string_agg(strings, NULL)| - +-------------------------+ - | abc| - +-------------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([Row(c1=["b", "a", "c"], c2="c")]) + >>> df.select(sf.array_append(df.c1, df.c2)).show() + +--------------------+ + |array_append(c1, c2)| + +--------------------+ + | [b, a, c, c]| + +--------------------+ - Example 2: Using string_agg function with a delimiter + Example 2: Appending a numeric value to an array column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',)], ['strings']) - >>> df.select(sf.string_agg('strings', ', ')).show() - +-----------------------+ - |string_agg(strings, , )| - +-----------------------+ - | a, b, c| - +-----------------------+ + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_append(df.data, 4)).show() + +---------------------+ + |array_append(data, 4)| + +---------------------+ + | [1, 2, 3, 4]| + +---------------------+ - Example 3: Using string_agg function with a binary column and delimiter + Example 3: Appending a null value to an array column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',)], ['bytes']) - >>> df.select(sf.string_agg('bytes', b'\x42')).show() + >>> df = spark.createDataFrame([([1, 2, 3],)], ['data']) + >>> df.select(sf.array_append(df.data, None)).show() +------------------------+ - |string_agg(bytes, X'42')| + |array_append(data, NULL)| +------------------------+ - | [01 42 02 42 03]| + | [1, 2, 3, NULL]| +------------------------+ - Example 4: Using string_agg function on a column with all None values + Example 4: Appending a value to a NULL array column >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.string_agg('strings')).show() - +-------------------------+ - |string_agg(strings, NULL)| - +-------------------------+ - | NULL| - +-------------------------+ + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([(None,)], schema=schema) + >>> df.select(sf.array_append(df.data, 4)).show() + +---------------------+ + |array_append(data, 4)| + +---------------------+ + | NULL| + +---------------------+ + + Example 5: Appending a value to an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_append(df.data, 1)).show() + +---------------------+ + |array_append(data, 1)| + +---------------------+ + | [1]| + +---------------------+ """ - if delimiter is None: - return _invoke_function_over_columns("string_agg", col) - else: - return _invoke_function_over_columns("string_agg", col, lit(delimiter)) + return _invoke_function_over_columns("array_append", col, lit(value)) @_try_remote_functions -def string_agg_distinct( - col: "ColumnOrName", delimiter: Optional[Union[Column, str, bytes]] = None -) -> Column: +def explode(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the concatenation of distinct non-null input values, - separated by the delimiter. + Returns a new row for each element in the given array or map. + Uses the default column name `col` for elements in the array and + `key` and `value` for elements in the map unless specified otherwise. - An alias of :func:`listagg_distinct`. + .. versionadded:: 1.4.0 - .. versionadded:: 4.0.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - delimiter : :class:`~pyspark.sql.Column`, literal string or bytes, optional - the delimiter to separate the values. The default value is None. + Target column to work on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - the column for computed results. + One row per array item or map key value. + Returns a column of the element type of the input array, or the key and value + columns of the input map. - Examples + See Also -------- - Example 1: Using string_agg_distinct function + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.string_agg_distinct('strings')).show() - +----------------------------------+ - |string_agg(DISTINCT strings, NULL)| - +----------------------------------+ - | abc| - +----------------------------------+ + Notes + ----- + Only one explode is allowed per SELECT clause. - Example 2: Using string_agg_distinct function with a delimiter + Examples + -------- + Example 1: Exploding an array column >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('a',), ('b',), (None,), ('c',), ('b',)], ['strings']) - >>> df.select(sf.string_agg_distinct('strings', ', ')).show() - +--------------------------------+ - |string_agg(DISTINCT strings, , )| - +--------------------------------+ - | a, b, c| - +--------------------------------+ - - Example 3: Using string_agg_distinct function with a binary column and delimiter + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.show() + +---+---------------+ + | i| a| + +---+---------------+ + | 1|[1, 2, 3, NULL]| + | 2| []| + | 3| NULL| + +---+---------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(b'\x01',), (b'\x02',), (None,), (b'\x03',), (b'\x02',)], - ... ['bytes']) - >>> df.select(sf.string_agg_distinct('bytes', b'\x42')).show() - +---------------------------------+ - |string_agg(DISTINCT bytes, X'42')| - +---------------------------------+ - | [01 42 02 42 03]| - +---------------------------------+ + >>> df.select('*', sf.explode('a')).show() + +---+---------------+----+ + | i| a| col| + +---+---------------+----+ + | 1|[1, 2, 3, NULL]| 1| + | 1|[1, 2, 3, NULL]| 2| + | 1|[1, 2, 3, NULL]| 3| + | 1|[1, 2, 3, NULL]|NULL| + +---+---------------+----+ - Example 4: Using string_agg_distinct function on a column with all None values + Example 2: Exploding a map column >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import StructType, StructField, StringType - >>> schema = StructType([StructField("strings", StringType(), True)]) - >>> df = spark.createDataFrame([(None,), (None,), (None,), (None,)], schema=schema) - >>> df.select(sf.string_agg_distinct('strings')).show() - +----------------------------------+ - |string_agg(DISTINCT strings, NULL)| - +----------------------------------+ - | NULL| - +----------------------------------+ - """ - if delimiter is None: - return _invoke_function_over_columns("string_agg_distinct", col) - else: - return _invoke_function_over_columns("string_agg_distinct", col, lit(delimiter)) - - -@_try_remote_functions -def product(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns the product of the values in a group. + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.show(truncate=False) + +---+---------------------------+ + |i |m | + +---+---------------------------+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| + |2 |{} | + |3 |NULL | + +---+---------------------------+ - .. versionadded:: 3.2.0 + >>> df.select('*', sf.explode('m')).show(truncate=False) + +---+---------------------------+---+-----+ + |i |m |key|value| + +---+---------------------------+---+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | + +---+---------------------------+---+-----+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Exploding multiple array columns - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - column containing values to be multiplied together + >>> import pyspark.sql.functions as sf + >>> df = spark.sql('SELECT ARRAY(1,2) AS a1, ARRAY(3,4,5) AS a2') + >>> df.select( + ... '*', sf.explode('a1').alias('v1') + ... ).select('*', sf.explode('a2').alias('v2')).show() + +------+---------+---+---+ + | a1| a2| v1| v2| + +------+---------+---+---+ + |[1, 2]|[3, 4, 5]| 1| 3| + |[1, 2]|[3, 4, 5]| 1| 4| + |[1, 2]|[3, 4, 5]| 1| 5| + |[1, 2]|[3, 4, 5]| 2| 3| + |[1, 2]|[3, 4, 5]| 2| 4| + |[1, 2]|[3, 4, 5]| 2| 5| + +------+---------+---+---+ - Returns - ------- - :class:`~pyspark.sql.Column` or column name - the column for computed results. + Example 4: Exploding an array of struct column - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT id % 3 AS mod3, id AS value FROM RANGE(10)") - >>> df.groupBy('mod3').agg(sf.product('value')).orderBy('mod3').show() - +----+--------------+ - |mod3|product(value)| - +----+--------------+ - | 0| 0.0| - | 1| 28.0| - | 2| 80.0| - +----+--------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select(sf.explode('a').alias("s")).select("s.*").show() + +---+---+ + | a| b| + +---+---+ + | 1| 2| + | 3| 4| + +---+---+ """ - return _invoke_function_over_columns("product", col) + return _invoke_function_over_columns("explode", col) @_try_remote_functions -def stddev(col: "ColumnOrName") -> Column: +def posexplode(col: "ColumnOrName") -> Column: """ - Aggregate function: alias for stddev_samp. + Returns a new row for each element with position in the given array or map. + Uses the default column name `pos` for position, and `col` for elements in the + array and `key` and `value` for elements in the map unless specified otherwise. - .. versionadded:: 1.6.0 + .. versionadded:: 2.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -21700,170 +22190,199 @@ def stddev(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. - - See Also - -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev_pop` - :meth:`pyspark.sql.functions.stddev_samp` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.skewness` - :meth:`pyspark.sql.functions.kurtosis` + target column to work on. Returns ------- :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. + one row per array item or map key value including positions as a separate column. - Examples + See Also -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.stddev("id")).show() - +------------------+ - | stddev(id)| - +------------------+ - |1.8708286933869...| - +------------------+ - """ - return _invoke_function_over_columns("stddev", col) - + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` -@_try_remote_functions -def std(col: "ColumnOrName") -> Column: - """ - Aggregate function: alias for stddev_samp. + Examples + -------- + Example 1: Exploding an array column - .. versionadded:: 3.5.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.show() + +---+---------------+ + | i| a| + +---+---------------+ + | 1|[1, 2, 3, NULL]| + | 2| []| + | 3| NULL| + +---+---------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + >>> df.select('*', sf.posexplode('a')).show() + +---+---------------+---+----+ + | i| a|pos| col| + +---+---------------+---+----+ + | 1|[1, 2, 3, NULL]| 0| 1| + | 1|[1, 2, 3, NULL]| 1| 2| + | 1|[1, 2, 3, NULL]| 2| 3| + | 1|[1, 2, 3, NULL]| 3|NULL| + +---+---------------+---+----+ - Returns - ------- - :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. + Example 2: Exploding a map column - See Also - -------- - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.stddev_pop` - :meth:`pyspark.sql.functions.stddev_samp` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.skewness` - :meth:`pyspark.sql.functions.kurtosis` + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.show(truncate=False) + +---+---------------------------+ + |i |m | + +---+---------------------------+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| + |2 |{} | + |3 |NULL | + +---+---------------------------+ - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.std("id")).show() - +------------------+ - | std(id)| - +------------------+ - |1.8708286933869...| - +------------------+ + >>> df.select('*', sf.posexplode('m')).show(truncate=False) + +---+---------------------------+---+---+-----+ + |i |m |pos|key|value| + +---+---------------------------+---+---+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | + +---+---------------------------+---+---+-----+ """ - return _invoke_function_over_columns("std", col) + return _invoke_function_over_columns("posexplode", col) @_try_remote_functions -def stddev_samp(col: "ColumnOrName") -> Column: +def inline(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the unbiased sample standard deviation of - the expression in a group. + Explodes an array of structs into a table. - .. versionadded:: 1.6.0 + This function takes an input column containing an array of structs and returns a + new column where each struct in the array is exploded into a separate row. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.4.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + Input column of values to explode. Returns ------- :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. + Generator expression with the inline exploded result. See Also -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.stddev_pop` - :meth:`pyspark.sql.functions.var_samp` + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline_outer` Examples -------- + Example 1: Using inline with a single struct array column + >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.stddev_samp("id")).show() - +------------------+ - | stddev_samp(id)| - +------------------+ - |1.8708286933869...| - +------------------+ - """ - return _invoke_function_over_columns("stddev_samp", col) + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select('*', sf.inline(df.a)).show() + +----------------+---+---+ + | a| a| b| + +----------------+---+---+ + |[{1, 2}, {3, 4}]| 1| 2| + |[{1, 2}, {3, 4}]| 3| 4| + +----------------+---+---+ + Example 2: Using inline with a column name -@_try_remote_functions -def stddev_pop(col: "ColumnOrName") -> Column: - """ - Aggregate function: returns population standard deviation of - the expression in a group. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select('*', sf.inline('a')).show() + +----------------+---+---+ + | a| a| b| + +----------------+---+---+ + |[{1, 2}, {3, 4}]| 1| 2| + |[{1, 2}, {3, 4}]| 3| 4| + +----------------+---+---+ - .. versionadded:: 1.6.0 + Example 3: Using inline with an alias - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') + >>> df.select('*', sf.inline('a').alias("c1", "c2")).show() + +----------------+---+---+ + | a| c1| c2| + +----------------+---+---+ + |[{1, 2}, {3, 4}]| 1| 2| + |[{1, 2}, {3, 4}]| 3| 4| + +----------------+---+---+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + Example 4: Using inline with multiple struct array columns - Returns - ------- - :class:`~pyspark.sql.Column` - standard deviation of given column. - Returns a column that evaluates to a double. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a1, ARRAY(NAMED_STRUCT("c",5,"d",6), NAMED_STRUCT("c",7,"d",8)) AS a2') + >>> df.select( + ... '*', sf.inline('a1') + ... ).select('*', sf.inline('a2')).show() + +----------------+----------------+---+---+---+---+ + | a1| a2| a| b| c| d| + +----------------+----------------+---+---+---+---+ + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 5| 6| + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 7| 8| + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 5| 6| + |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 7| 8| + +----------------+----------------+---+---+---+---+ - See Also - -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.stddev_samp` - :meth:`pyspark.sql.functions.var_pop` + Example 5: Using inline with a nested struct array column - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.range(6).select(sf.stddev_pop("id")).show() - +-----------------+ - | stddev_pop(id)| - +-----------------+ - |1.707825127659...| - +-----------------+ + >>> df = spark.sql('SELECT NAMED_STRUCT("a",1,"b",2,"c",ARRAY(NAMED_STRUCT("c",3,"d",4), NAMED_STRUCT("c",5,"d",6))) AS s') + >>> df.select('*', sf.inline('s.c')).show(truncate=False) + +------------------------+---+---+ + |s |c |d | + +------------------------+---+---+ + |{1, 2, [{3, 4}, {5, 6}]}|3 |4 | + |{1, 2, [{3, 4}, {5, 6}]}|5 |6 | + +------------------------+---+---+ + + Example 6: Using inline with a column containing: array continaing null, empty array and null + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') + >>> df.show(truncate=False) + +---+----------------------+ + |i |s | + +---+----------------------+ + |1 |[{1, 2}, NULL, {3, 4}]| + |2 |[] | + |3 |NULL | + +---+----------------------+ + + >>> df.select('*', sf.inline('s')).show(truncate=False) + +---+----------------------+----+----+ + |i |s |a |b | + +---+----------------------+----+----+ + |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | + |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| + |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | + +---+----------------------+----+----+ """ - return _invoke_function_over_columns("stddev_pop", col) + return _invoke_function_over_columns("inline", col) @_try_remote_functions -def variance(col: "ColumnOrName") -> Column: +def explode_outer(col: "ColumnOrName") -> Column: """ - Aggregate function: alias for var_samp + Returns a new row for each element in the given array or map. + Unlike explode, if the array/map is null or empty then null is produced. + Uses the default column name `col` for elements in the array and + `key` and `value` for elements in the map unless specified otherwise. - .. versionadded:: 1.6.0 + .. versionadded:: 2.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -21871,44 +22390,69 @@ def variance(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + target column to work on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - variance of given column. + one row per array item or map key value. + Returns a column of the element type of the input array, or the key and value + columns of the input map. See Also -------- - :meth:`pyspark.sql.functions.var_pop` - :meth:`pyspark.sql.functions.var_samp` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.skewness` - :meth:`pyspark.sql.functions.kurtosis` - :meth:`pyspark.sql.functions.std` + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` Examples -------- + Example 1: Using an array column + >>> from pyspark.sql import functions as sf - >>> df = spark.range(6) - >>> df.select(sf.variance(df.id)).show() - +------------+ - |variance(id)| - +------------+ - | 3.5| - +------------+ + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.select('*', sf.explode_outer('a')).show() + +---+---------------+----+ + | i| a| col| + +---+---------------+----+ + | 1|[1, 2, 3, NULL]| 1| + | 1|[1, 2, 3, NULL]| 2| + | 1|[1, 2, 3, NULL]| 3| + | 1|[1, 2, 3, NULL]|NULL| + | 2| []|NULL| + | 3| NULL|NULL| + +---+---------------+----+ + + Example 2: Using a map column + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.select('*', sf.explode_outer('m')).show(truncate=False) + +---+---------------------------+----+-----+ + |i |m |key |value| + +---+---------------------------+----+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | + |2 |{} |NULL|NULL | + |3 |NULL |NULL|NULL | + +---+---------------------------+----+-----+ """ - return _invoke_function_over_columns("variance", col) + return _invoke_function_over_columns("explode_outer", col) @_try_remote_functions -def var_samp(col: "ColumnOrName") -> Column: +def posexplode_outer(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the unbiased sample variance of - the values in a group. + Returns a new row for each element with position in the given array or map. + Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced. + Uses the default column name `pos` for position, and `col` for elements in the + array and `key` and `value` for elements in the map unless specified otherwise. - .. versionadded:: 1.6.0 + .. versionadded:: 2.3.0 .. versionchanged:: 3.4.0 Supports Spark Connect. @@ -21916,4309 +22460,4118 @@ def var_samp(col: "ColumnOrName") -> Column: Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + target column to work on. Returns ------- :class:`~pyspark.sql.Column` - variance of given column. + one row per array item or map key value including positions as a separate column. See Also -------- - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.var_pop` - :meth:`pyspark.sql.functions.stddev_samp` + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.inline` + :meth:`pyspark.sql.functions.inline_outer` Examples -------- + Example 1: Using an array column + >>> from pyspark.sql import functions as sf - >>> df = spark.range(6) - >>> df.select(sf.var_samp(df.id)).show() - +------------+ - |var_samp(id)| - +------------+ - | 3.5| - +------------+ + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') + >>> df.select('*', sf.posexplode_outer('a')).show() + +---+---------------+----+----+ + | i| a| pos| col| + +---+---------------+----+----+ + | 1|[1, 2, 3, NULL]| 0| 1| + | 1|[1, 2, 3, NULL]| 1| 2| + | 1|[1, 2, 3, NULL]| 2| 3| + | 1|[1, 2, 3, NULL]| 3|NULL| + | 2| []|NULL|NULL| + | 3| NULL|NULL|NULL| + +---+---------------+----+----+ + + Example 2: Using a map column + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') + >>> df.select('*', sf.posexplode_outer('m')).show(truncate=False) + +---+---------------------------+----+----+-----+ + |i |m |pos |key |value| + +---+---------------------------+----+----+-----+ + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | + |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | + |2 |{} |NULL|NULL|NULL | + |3 |NULL |NULL|NULL|NULL | + +---+---------------------------+----+----+-----+ """ - return _invoke_function_over_columns("var_samp", col) + return _invoke_function_over_columns("posexplode_outer", col) @_try_remote_functions -def var_pop(col: "ColumnOrName") -> Column: +def inline_outer(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the population variance of the values in a group. - - .. versionadded:: 1.6.0 + Explodes an array of structs into a table. + Unlike inline, if the array is null or empty then null is produced for each nested column. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.4.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. + input column of values to explode. Returns ------- :class:`~pyspark.sql.Column` - variance of given column. + generator expression with the inline exploded result. See Also -------- - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.var_samp` - :meth:`pyspark.sql.functions.stddev_pop` + :meth:`pyspark.sql.functions.explode` + :meth:`pyspark.sql.functions.explode_outer` + :meth:`pyspark.sql.functions.posexplode` + :meth:`pyspark.sql.functions.posexplode_outer` + :meth:`pyspark.sql.functions.inline` + + Notes + ----- + Supports Spark Connect. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.range(6) - >>> df.select(sf.var_pop(df.id)).show() - +------------------+ - | var_pop(id)| - +------------------+ - |2.9166666666666...| - +------------------+ + >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') + >>> df.printSchema() + root + |-- i: integer (nullable = false) + |-- s: array (nullable = true) + | |-- element: struct (containsNull = true) + | | |-- a: integer (nullable = false) + | | |-- b: integer (nullable = false) + + >>> df.select('*', sf.inline_outer('s')).show(truncate=False) + +---+----------------------+----+----+ + |i |s |a |b | + +---+----------------------+----+----+ + |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | + |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| + |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | + |2 |[] |NULL|NULL| + |3 |NULL |NULL|NULL| + +---+----------------------+----+----+ """ - return _invoke_function_over_columns("var_pop", col) + return _invoke_function_over_columns("inline_outer", col) @_try_remote_functions -def regr_avgx(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def get_json_object(col: "ColumnOrName", path: str) -> Column: """ - Aggregate function: returns the average of the independent variable for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Extracts json object from a json string based on json `path` specified, and returns json string + of the extracted json object. It will return null if the input json string is invalid. - .. versionadded:: 3.5.0 + .. versionadded:: 1.6.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + string column in json format. + A column that evaluates to a string. + path : str + path to the json object to extract. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the average of the independent variable for non-null pairs in a group. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + string representation of given JSON object value. + Returns a column that evaluates to a string. Examples -------- - Example 1: All pairs are non-null + Example 1: Extract a json object from json string - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | 2.75| 2.75| - +---------------+------+ + >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] + >>> df = spark.createDataFrame(data, ("key", "jstring")) + >>> df.select(df.key, + ... get_json_object(df.jstring, '$.f1').alias("c0"), + ... get_json_object(df.jstring, '$.f2').alias("c1") + ... ).show() + +---+-------+------+ + |key| c0| c1| + +---+-------+------+ + | 1| value1|value2| + | 2|value12| NULL| + +---+-------+------+ - Example 2: All pairs' x values are null + Example 2: Extract a json object from json array - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | NULL| NULL| - +---------------+------+ + >>> data = [ + ... ("1", '''[{"f1": "value1"},{"f1": "value2"}]'''), + ... ("2", '''[{"f1": "value12"},{"f2": "value13"}]''') + ... ] + >>> df = spark.createDataFrame(data, ("key", "jarray")) + >>> df.select(df.key, + ... get_json_object(df.jarray, '$[0].f1').alias("c0"), + ... get_json_object(df.jarray, '$[1].f2').alias("c1") + ... ).show() + +---+-------+-------+ + |key| c0| c1| + +---+-------+-------+ + | 1| value1| NULL| + | 2|value12|value13| + +---+-------+-------+ - Example 3: All pairs' y values are null + >>> df.select(df.key, + ... get_json_object(df.jarray, '$[*].f1').alias("c0"), + ... get_json_object(df.jarray, '$[*].f2').alias("c1") + ... ).show() + +---+-------------------+---------+ + |key| c0| c1| + +---+-------------------+---------+ + | 1|["value1","value2"]| NULL| + | 2| "value12"|"value13"| + +---+-------------------+---------+ + """ + from pyspark.sql.classic.column import _to_java_column - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | NULL| 1.0| - +---------------+------+ + return _invoke_function("get_json_object", _to_java_column(col), _enum_to_value(path)) - Example 4: Some pairs' x values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | 3.0| 3.0| - +---------------+------+ +@_try_remote_functions +def json_tuple(col: "ColumnOrName", *fields: str) -> Column: + """Creates a new row for a json column according to the given field names. - Example 5: Some pairs' x or y values are null + .. versionadded:: 1.6.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgx("y", "x"), sf.avg("x")).show() - +---------------+------+ - |regr_avgx(y, x)|avg(x)| - +---------------+------+ - | 3.0| 3.0| - +---------------+------+ + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + string column in json format + A column that evaluates to a string. + fields : str + a field or fields to extract + Each a column that evaluates to a string. + + Returns + ------- + :class:`~pyspark.sql.Column` + a new row for each given field value from json object + Returns a column that evaluates to a string. + + Examples + -------- + >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] + >>> df = spark.createDataFrame(data, ("key", "jstring")) + >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect() + [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)] """ - return _invoke_function_over_columns("regr_avgx", y, x) + from pyspark.sql.classic.column import _to_java_column, _to_seq + + if len(fields) == 0: + raise PySparkValueError( + errorClass="CANNOT_BE_EMPTY", + messageParameters={"item": "field"}, + ) + sc = _get_active_spark_context() + return _invoke_function("json_tuple", _to_java_column(col), _to_seq(sc, fields)) @_try_remote_functions -def regr_avgy(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def from_json( + col: "ColumnOrName", + schema: Union[ArrayType, StructType, MapType, Column, str], + options: Optional[Mapping[str, str]] = None, +) -> Column: """ - Aggregate function: returns the average of the dependent variable for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType` + as keys type, :class:`StructType` or :class:`ArrayType` with + the specified schema. Returns `null`, in the case of an unparsable string. - .. versionadded:: 3.5.0 + .. versionadded:: 2.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + a column or column name in JSON format + schema : :class:`StructType`, :class:`ArrayType`, :class:`MapType`, or str + a StructType, ArrayType of StructType, MapType, or Python string literal with a DDL-formatted string + A column that evaluates to a string, or a DDL-formatted type string, or a DataType. + to use when parsing the json column + options : dict, optional + options to control parsing. accepts the same options as the json datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - the average of the dependent variable for non-null pairs in a group. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + a new column of complex type from given JSON object. + Returns a column that evaluates to a struct, array, or map. Examples -------- - Example 1: All pairs are non-null + Example 1: Parsing JSON with a specified schema >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------+ - |regr_avgy(y, x)|avg(y)| - +---------------+------+ - | 1.75| 1.75| - +---------------+------+ + >>> from pyspark.sql.types import StructType, StructField, IntegerType + >>> schema = StructType([StructField("a", IntegerType())]) + >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, schema).alias("json")).show() + +----+ + |json| + +----+ + | {1}| + +----+ - Example 2: All pairs' x values are null + Example 2: Parsing JSON with a DDL-formatted string. >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------+ - |regr_avgy(y, x)|avg(y)| - +---------------+------+ - | NULL| 1.0| - +---------------+------+ + >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, "a INT").alias("json")).show() + +----+ + |json| + +----+ + | {1}| + +----+ - Example 3: All pairs' y values are null + Example 3: Parsing JSON into a MapType >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------+ - |regr_avgy(y, x)|avg(y)| - +---------------+------+ - | NULL| NULL| - +---------------+------+ + >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, "MAP").alias("json")).show() + +--------+ + | json| + +--------+ + |{a -> 1}| + +--------+ - Example 4: Some pairs' x values are null + Example 4: Parsing JSON into an ArrayType of StructType >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +------------------+------+ - | regr_avgy(y, x)|avg(y)| - +------------------+------+ - |1.6666666666666...| 1.75| - +------------------+------+ + >>> from pyspark.sql.types import ArrayType, StructType, StructField, IntegerType + >>> schema = ArrayType(StructType([StructField("a", IntegerType())])) + >>> df = spark.createDataFrame([(1, '''[{"a": 1}]''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, schema).alias("json")).show() + +-----+ + | json| + +-----+ + |[{1}]| + +-----+ - Example 5: Some pairs' x or y values are null + Example 5: Parsing JSON into an ArrayType >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_avgy("y", "x"), sf.avg("y")).show() - +---------------+------------------+ - |regr_avgy(y, x)| avg(y)| - +---------------+------------------+ - | 1.5|1.6666666666666...| - +---------------+------------------+ + >>> from pyspark.sql.types import ArrayType, IntegerType + >>> schema = ArrayType(IntegerType()) + >>> df = spark.createDataFrame([(1, '''[1, 2, 3]''')], ("key", "value")) + >>> df.select(sf.from_json(df.value, schema).alias("json")).show() + +---------+ + | json| + +---------+ + |[1, 2, 3]| + +---------+ + + Example 6: Parsing JSON with specified options + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, '''{a:123}'''), (2, '''{"a":456}''')], ("key", "value")) + >>> parsed1 = sf.from_json(df.value, "a INT") + >>> parsed2 = sf.from_json(df.value, "a INT", {"allowUnquotedFieldNames": "true"}) + >>> df.select("value", parsed1, parsed2).show() + +---------+----------------+----------------+ + | value|from_json(value)|from_json(value)| + +---------+----------------+----------------+ + | {a:123}| {NULL}| {123}| + |{"a":456}| {456}| {456}| + +---------+----------------+----------------+ """ - return _invoke_function_over_columns("regr_avgy", y, x) + from pyspark.sql.classic.column import _to_java_column + + if isinstance(schema, DataType): + schema = schema.json() + elif isinstance(schema, Column): + schema = _to_java_column(schema) + return _invoke_function("from_json", _to_java_column(col), schema, _options_to_str(options)) @_try_remote_functions -def regr_count(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def try_parse_json( + col: "ColumnOrName", +) -> Column: """ - Aggregate function: returns the number of non-null number pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Parses a column containing a JSON string into a :class:`VariantType`. Returns None if a string + contains an invalid JSON value. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + col : :class:`~pyspark.sql.Column` or str + a column or column name JSON formatted strings. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - the number of non-null number pairs in a group. + a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, 2), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 4| 4| - +----------------+--------+ - - Example 2: All pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 0| 1| - +----------------+--------+ - - Example 3: All pairs' y values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 0| 1| - +----------------+--------+ - - Example 4: Some pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (2, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 3| 4| - +----------------+--------+ - - Example 5: Some pairs' x or y values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 2), (2, null), (null, 3), (2, 4) AS tab(y, x)") - >>> df.select(sf.regr_count("y", "x"), sf.count(sf.lit(0))).show() - +----------------+--------+ - |regr_count(y, x)|count(0)| - +----------------+--------+ - | 2| 4| - +----------------+--------+ + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''}, {'json': '''{a : 1}'''} ]) + >>> df.select(to_json(try_parse_json(df.json))).collect() + [Row(to_json(try_parse_json(json))='{"a":1}'), Row(to_json(try_parse_json(json))=None)] """ - return _invoke_function_over_columns("regr_count", y, x) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("try_parse_json", _to_java_column(col)) @_try_remote_functions -def regr_intercept(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def to_variant_object( + col: "ColumnOrName", +) -> Column: """ - Aggregate function: returns the intercept of the univariate linear regression line - for non-null pairs in a group, where `y` is the dependent variable and - `x` is the independent variable. + Converts a column containing nested inputs (array/map/struct) into a variants where maps and + structs are converted to variant objects which are unordered unlike SQL structs. Input maps can + only have string keys. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + a column with a nested schema or column name + A column that evaluates to an array, map, or struct. Returns ------- :class:`~pyspark.sql.Column` - the intercept of the univariate linear regression line for non-null pairs in a group. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + a new column of VariantType. + Returns a column that evaluates to a variant. Examples -------- - Example 1: All pairs are non-null + Example 1: Converting an array containing a nested struct into a variant - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, StructType, StructField, StringType, MapType + >>> schema = StructType([ + ... StructField("i", StringType(), True), + ... StructField("v", ArrayType(StructType([ + ... StructField("a", MapType(StringType(), StringType()), True) + ... ]), True)) + ... ]) + >>> data = [("1", [{"a": {"b": 2}}])] + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.to_variant_object(df.v)) + DataFrame[to_variant_object(v): variant] + >>> df.select(sf.to_variant_object(df.v)).show(truncate=False) +--------------------+ - |regr_intercept(y, x)| + |to_variant_object(v)| +--------------------+ - | 0.0| + |[{"a":{"b":"2"}}] | +--------------------+ + """ + from pyspark.sql.classic.column import _to_java_column - Example 2: All pairs' x values are null + return _invoke_function("to_variant_object", _to_java_column(col)) - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | NULL| - +--------------------+ - Example 3: All pairs' y values are null +@_try_remote_functions +def variant_from_arrays(keys: "ColumnOrName", values: "ColumnOrName") -> Column: + """ + Creates a variant object from the given arrays of keys and values. The keys must be non-null + strings and the two arrays must have the same length. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | NULL| - +--------------------+ + .. versionadded:: 4.4.0 - Example 4: Some pairs' x values are null + Parameters + ---------- + keys : :class:`~pyspark.sql.Column` or column name + an array of string keys. + values : :class:`~pyspark.sql.Column` or column name + an array of values. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | 0.0| - +--------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. - Example 5: Some pairs' x or y values are null + See Also + -------- + :meth:`pyspark.sql.functions.variant_from_entries` + :meth:`pyspark.sql.functions.to_variant_object` - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_intercept("y", "x")).show() - +--------------------+ - |regr_intercept(y, x)| - +--------------------+ - | 0.0| - +--------------------+ + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT array('a', 'b') AS keys, array(1, 2) AS values") + >>> df.select(sf.variant_from_arrays("keys", "values").cast("string").alias("r")).collect() + [Row(r='{"a":1,"b":2}')] """ - return _invoke_function_over_columns("regr_intercept", y, x) + return _invoke_function_over_columns("variant_from_arrays", keys, values) @_try_remote_functions -def regr_r2(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def variant_from_entries(entries: "ColumnOrName") -> Column: """ - Aggregate function: returns the coefficient of determination for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Creates a variant object from an array of key/value struct entries. The keys must be non-null + strings. - .. versionadded:: 3.5.0 + .. versionadded:: 4.4.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + entries : :class:`~pyspark.sql.Column` or column name + an array of key/value structs, where the first field is a string key and the second field + is the value. Returns ------- :class:`~pyspark.sql.Column` - the coefficient of determination for non-null pairs in a group. + a new column of VariantType. See Also -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + :meth:`pyspark.sql.functions.variant_from_arrays` + :meth:`pyspark.sql.functions.to_variant_object` Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | 1.0| - +-------------+ - - Example 2: All pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | NULL| - +-------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT array(struct('a', 1), struct('b', 2)) AS entries") + >>> df.select(sf.variant_from_entries("entries").cast("string").alias("r")).collect() + [Row(r='{"a":1,"b":2}')] + """ + return _invoke_function_over_columns("variant_from_entries", entries) - Example 3: All pairs' y values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | NULL| - +-------------+ +@_try_remote_functions +def parse_json( + col: "ColumnOrName", +) -> Column: + """ + Parses a column containing a JSON string into a :class:`VariantType`. Throws exception if a + string represents an invalid JSON value. - Example 4: Some pairs' x values are null + .. versionadded:: 4.0.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | 1.0| - +-------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + a column or column name JSON formatted strings. + A column that evaluates to a string. - Example 5: Some pairs' x or y values are null + Returns + ------- + :class:`~pyspark.sql.Column` + a new column of VariantType. + Returns a column that evaluates to a variant. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_r2("y", "x")).show() - +-------------+ - |regr_r2(y, x)| - +-------------+ - | 1.0| - +-------------+ + Examples + -------- + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(to_json(parse_json(df.json))).collect() + [Row(to_json(parse_json(json))='{"a":1}')] """ - return _invoke_function_over_columns("regr_r2", y, x) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("parse_json", _to_java_column(col)) @_try_remote_functions -def regr_slope(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def is_variant_null(v: "ColumnOrName") -> Column: """ - Aggregate function: returns the slope of the linear regression line for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Check if a variant value is a variant null. Returns true if and only if the input is a variant + null and false otherwise (including in the case of SQL NULL). - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. Returns ------- :class:`~pyspark.sql.Column` - the slope of the linear regression line for non-null pairs in a group. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` + a boolean column indicating whether the variant value is a variant null + Returns a column that evaluates to a boolean. Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | 1.0| - +----------------+ - - Example 2: All pairs' x values are null + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(is_variant_null(parse_json(df.json)).alias("r")).collect() + [Row(r=False)] + """ + from pyspark.sql.classic.column import _to_java_column - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | NULL| - +----------------+ + return _invoke_function("is_variant_null", _to_java_column(v)) - Example 3: All pairs' y values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | NULL| - +----------------+ +@_try_remote_functions +def is_valid_variant(v: "ColumnOrName") -> Column: + """ + Check if a variant value is valid. Returns true if the variant is valid, false if it is + malformed, and NULL if the input is NULL. - Example 4: Some pairs' x values are null + .. versionadded:: 4.2.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | 1.0| - +----------------+ + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. - Example 5: Some pairs' x or y values are null + Returns + ------- + :class:`~pyspark.sql.Column` + a boolean column indicating whether the variant value is valid + Returns a column that evaluates to a boolean. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_slope("y", "x")).show() - +----------------+ - |regr_slope(y, x)| - +----------------+ - | 1.0| - +----------------+ + Examples + -------- + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(is_valid_variant(parse_json(df.json)).alias("r")).collect() + [Row(r=True)] """ - return _invoke_function_over_columns("regr_slope", y, x) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("is_valid_variant", _to_java_column(v)) @_try_remote_functions -def regr_sxx(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def variant_delete(v: "ColumnOrName", *paths: Union[Column, str]) -> Column: """ - Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Removes fields or array elements from a variant at the given JSONPath locations. + Multiple paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are + skipped. - .. versionadded:: 3.5.0 + .. versionadded:: 5.0.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + paths : :class:`~pyspark.sql.Column` or str + one or more JSONPath deletion targets. A `str` is a literal path; a + :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path + should start with `$` and is followed by one or more segments like + `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not + A column that evaluates to a string. + allowed. Returns ------- :class:`~pyspark.sql.Column` - REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group. + a variant column with the specified paths removed + Returns a column that evaluates to a variant. - See Also + Examples -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_sxy` - :meth:`pyspark.sql.functions.regr_syy` - - Examples - -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | 5.0| - +--------------+ - - Example 2: All pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | NULL| - +--------------+ - - Example 3: All pairs' y values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | NULL| - +--------------+ - - Example 4: Some pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +-----------------+ - | regr_sxx(y, x)| - +-----------------+ - |4.666666666666...| - +-----------------+ + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_delete + >>> df = spark.createDataFrame([{ + ... 'json': '''{ "a" : 1, "b" : 2, "c" : 3, "items" : [1, 2, 3] }''', + ... 'path': '$.a' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_delete(v, lit(None), "$.a", "$.c")).alias("r")).collect() + [Row(r='{"b":2,"items":[1,2,3]}')] + >>> df.select(to_json(variant_delete(v, "$.missing")).alias("r")).collect() + [Row(r='{"a":1,"b":2,"c":3,"items":[1,2,3]}')] + >>> df.select(to_json(variant_delete(v, df.path)).alias("r")).collect() + [Row(r='{"b":2,"c":3,"items":[1,2,3]}')] + >>> df.select(to_json(variant_delete(v, "$.items[0]", "$.items[0]")).alias("r")).collect() + [Row(r='{"a":1,"b":2,"c":3,"items":[3]}')] + >>> df.select(variant_delete(lit(None), "$.a").alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column, _to_seq - Example 5: Some pairs' x or y values are null + if len(paths) == 0: + raise PySparkValueError( + errorClass="CANNOT_BE_EMPTY", + messageParameters={"item": "paths"}, + ) + sc = _get_active_spark_context() - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxx("y", "x")).show() - +--------------+ - |regr_sxx(y, x)| - +--------------+ - | 4.5| - +--------------+ - """ - return _invoke_function_over_columns("regr_sxx", y, x) + path_cols = [p if isinstance(p, Column) else lit(p) for p in paths] + return _invoke_function( + "variant_delete", + _to_java_column(v), + _to_java_column(path_cols[0]), + _to_seq(sc, path_cols[1:], _to_java_column), + ) @_try_remote_functions -def regr_sxy(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def variant_insert(v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName") -> Column: """ - Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Inserts a value into a variant at the given JSONPath location. An object path adds a new field + (error if it already exists); an array path inserts at the index, shifting later elements + right. Missing intermediate keys are created. Throws an error if a path segment hits a value + of an incompatible type. Returns NULL if any argument is NULL. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_syy` + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath insertion target. A `str` is a literal path; a + :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with + `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or + A column that evaluates to a string. + `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to insert. Any expression castable to variant. Returns ------- :class:`~pyspark.sql.Column` - REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group. + a variant column with `value` inserted at `path` + Returns a column that evaluates to a variant. Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | 5.0| - +--------------+ - - Example 2: All pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | NULL| - +--------------+ - - Example 3: All pairs' y values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | NULL| - +--------------+ - - Example 4: Some pairs' x values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +-----------------+ - | regr_sxy(y, x)| - +-----------------+ - |4.666666666666...| - +-----------------+ - - Example 5: Some pairs' x or y values are null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_sxy("y", "x")).show() - +--------------+ - |regr_sxy(y, x)| - +--------------+ - | 4.5| - +--------------+ + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_insert + >>> df = spark.createDataFrame([{ + ... 'json': '''{ "a": 1, "arr": ["x", "y"] }''', + ... 'path': '$.d' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_insert(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"b":2}')] + >>> df.select(to_json(variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] + >>> df.select(to_json(variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","z","y"]}')] + >>> df.select(to_json(variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] + >>> df.select(to_json(variant_insert(v, df.path, lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"d":9}')] + >>> df.select(to_json(variant_insert(v, "$.b", parse_json(lit('null')))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"b":null}')] + >>> df.select(variant_insert(v, "$.b", lit(None)).alias("r")).collect() + [Row(r=None)] """ - return _invoke_function_over_columns("regr_sxy", y, x) + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "variant_insert", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) @_try_remote_functions -def regr_syy(y: "ColumnOrName", x: "ColumnOrName") -> Column: +def try_variant_insert( + v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" +) -> Column: """ - Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs - in a group, where `y` is the dependent variable and `x` is the independent variable. + Inserts a value into a variant at the given JSONPath location. An object path adds a new field; + an array path inserts at the index, shifting later elements right. Missing intermediate keys + are created. Returns NULL if the field already exists or a path segment hits a value of an + incompatible type, or if any argument is NULL. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - y : :class:`~pyspark.sql.Column` or column name - the dependent variable. - A column that evaluates to a numeric. - x : :class:`~pyspark.sql.Column` or column name - the independent variable. - A column that evaluates to a numeric. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath insertion target. A `str` is a literal path; a + :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with + `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or + `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to insert. Any expression castable to variant. Returns ------- :class:`~pyspark.sql.Column` - REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group. - - See Also - -------- - :meth:`pyspark.sql.functions.regr_avgx` - :meth:`pyspark.sql.functions.regr_avgy` - :meth:`pyspark.sql.functions.regr_count` - :meth:`pyspark.sql.functions.regr_intercept` - :meth:`pyspark.sql.functions.regr_r2` - :meth:`pyspark.sql.functions.regr_slope` - :meth:`pyspark.sql.functions.regr_sxy` + a variant column with `value` inserted at `path`, or NULL if the insertion fails + Returns a column that evaluates to a variant. Examples -------- - Example 1: All pairs are non-null - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, 2), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | 5.0| - +--------------+ - - Example 2: All pairs' x values are null + >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_insert + >>> df = spark.createDataFrame([{'json': '''{ "a": 1, "arr": ["x", "y"] }'''}]) + >>> v = parse_json(df.json) + >>> df.select(to_json(try_variant_insert(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"b":2}')] + >>> df.select(to_json(try_variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] + >>> df.select(to_json(try_variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","z","y"]}')] + >>> df.select(to_json(try_variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() + [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] + >>> df.select(to_json(try_variant_insert(v, "$.a", lit(2))).alias("r")).collect() + [Row(r=None)] + >>> df.select(to_json(try_variant_insert(v, "$.a.b", lit(2))).alias("r")).collect() + [Row(r=None)] + >>> df.select(to_json(try_variant_insert(v, "$.b", lit(None))).alias("r")).collect() + [Row(r=None)] + """ + from pyspark.sql.classic.column import _to_java_column - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, null) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | NULL| - +--------------+ + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "try_variant_insert", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) - Example 3: All pairs' y values are null - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (null, 1) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | NULL| - +--------------+ +@_try_remote_functions +def variant_set( + v: "ColumnOrName", + path: Union[Column, str], + value: "ColumnOrName", + create_if_missing: bool = True, +) -> Column: + """ + Sets or upserts a value in a variant at the given JSONPath location. An existing object field + or array element at the target is replaced. A missing field, array index, or intermediate path + is created, unless `create_if_missing` is false, in which case the variant is left unchanged. + Throws an error if a path segment hits a value of an incompatible type. Returns NULL if any + argument is NULL. - Example 4: Some pairs' x values are null + .. versionadded:: 4.3.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (3, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +-----------------+ - | regr_syy(y, x)| - +-----------------+ - |4.666666666666...| - +-----------------+ + Parameters + ---------- + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies + the path at runtime. A valid path should start with `$` and is followed by one or more + A column that evaluates to a string. + segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to set. Any expression castable to variant. + create_if_missing : bool, optional + whether to create missing keys or out-of-range array indices (default True). + A column that evaluates to a boolean. Must be a constant. - Example 5: Some pairs' x or y values are null + Returns + ------- + :class:`~pyspark.sql.Column` + a variant column with `value` set at `path` + Returns a column that evaluates to a variant. - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT * FROM VALUES (1, 1), (2, null), (null, 3), (4, 4) AS tab(y, x)") - >>> df.select(sf.regr_syy("y", "x")).show() - +--------------+ - |regr_syy(y, x)| - +--------------+ - | 4.5| - +--------------+ + Examples + -------- + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_set + >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_set(v, "$.a", lit(9))).alias("r")).collect() + [Row(r='{"a":9,"arr":[1,2,3]}')] + >>> df.select(to_json(variant_set(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] + >>> df.select(to_json(variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,9,3]}')] + >>> df.select(to_json(variant_set(v, "$.b", lit(2), False)).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3]}')] + >>> df.select(to_json(variant_set(v, "$.a", parse_json(lit("null")))).alias("r")).collect() + [Row(r='{"a":null,"arr":[1,2,3]}')] + >>> df.select(to_json(variant_set(v, "$.a", lit(None))).alias("r")).collect() + [Row(r=None)] """ - return _invoke_function_over_columns("regr_syy", y, x) + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "variant_set", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + _enum_to_value(create_if_missing), + ) @_try_remote_functions -def every(col: "ColumnOrName") -> Column: +def try_variant_set( + v: "ColumnOrName", + path: Union[Column, str], + value: "ColumnOrName", + create_if_missing: bool = True, +) -> Column: """ - Aggregate function: returns true if all values of `col` are true. + Sets or upserts a value in a variant at the given JSONPath location. An existing object field + or array element at the target is replaced. A missing field, array index, or intermediate path + is created, unless `create_if_missing` is false, in which case the variant is left unchanged. + Returns NULL if a path segment hits a value of an incompatible type, or if any argument is NULL. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to check if all values are true. - A column that evaluates to a boolean. - - See Also - -------- - :meth:`pyspark.sql.functions.some` + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + path : :class:`~pyspark.sql.Column` or str + the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies + the path at runtime. A valid path should start with `$` and is followed by one or more + segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. + value : :class:`~pyspark.sql.Column` or str + the value to set. Any expression castable to variant. + create_if_missing : bool, optional + whether to create missing keys or out-of-range array indices (default True). Returns ------- :class:`~pyspark.sql.Column` - true if all values of `col` are true, false otherwise. + a variant column with `value` set at `path` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [True], [True]], ["flag"] - ... ).select(sf.every("flag")).show() - +-----------+ - |every(flag)| - +-----------+ - | true| - +-----------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [False], [True]], ["flag"] - ... ).select(sf.every("flag")).show() - +-----------+ - |every(flag)| - +-----------+ - | false| - +-----------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[False], [False], [False]], ["flag"] - ... ).select(sf.every("flag")).show() - +-----------+ - |every(flag)| - +-----------+ - | false| - +-----------+ + >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_set + >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) + >>> v = parse_json(df.json) + >>> df.select(to_json(try_variant_set(v, "$.a", lit(9))).alias("r")).collect() + [Row(r='{"a":9,"arr":[1,2,3]}')] + >>> df.select(to_json(try_variant_set(v, "$.b", lit(2))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] + >>> df.select(to_json(try_variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,9,3]}')] + >>> df.select(to_json(try_variant_set(v, "$.arr[5]", lit(9))).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3,null,null,9]}')] + >>> df.select(to_json(try_variant_set(v, "$.b", lit(2), False)).alias("r")).collect() + [Row(r='{"a":1,"arr":[1,2,3]}')] + >>> df.select(to_json(try_variant_set(v, "$.a.b", lit(9))).alias("r")).collect() + [Row(r=None)] + >>> df.select(to_json(try_variant_set(v, "$.a", lit(None))).alias("r")).collect() + [Row(r=None)] """ - return _invoke_function_over_columns("every", col) + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "try_variant_set", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + _enum_to_value(create_if_missing), + ) @_try_remote_functions -def bool_and(col: "ColumnOrName") -> Column: +def variant_array_append( + v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" +) -> Column: """ - Aggregate function: returns true if all values of `col` are true. + Appends a value to the array in a variant at the given JSONPath location. Returns the variant + unchanged if a path key or index is absent. Throws an error if a path segment hits a value of + an incompatible type or the target is not an array. Returns NULL if any argument is NULL. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to check if all values are true. - A column that evaluates to a boolean. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` + supplies the path at runtime. A valid path should start with `$` and is followed by zero or + A column that evaluates to a string. + more segments like `[123]`, `.name`, `['name']`, or `["name"]`. + value : :class:`~pyspark.sql.Column` or str + the value to append. Any expression castable to variant. Returns ------- :class:`~pyspark.sql.Column` - true if all values of `col` are true, false otherwise. - - See Also - -------- - :meth:`pyspark.sql.functions.bool_or` + a variant column with `value` appended to the array at `path` + Returns a column that evaluates to a variant. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) - >>> df.select(sf.bool_and("flag")).show() - +--------------+ - |bool_and(flag)| - +--------------+ - | true| - +--------------+ - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) - >>> df.select(sf.bool_and("flag")).show() - +--------------+ - |bool_and(flag)| - +--------------+ - | false| - +--------------+ - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) - >>> df.select(sf.bool_and("flag")).show() - +--------------+ - |bool_and(flag)| - +--------------+ - | false| - +--------------+ + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_array_append + >>> df = spark.createDataFrame([{ + ... 'json': '''[[1, 2], 5]''', + ... 'path': '$[0]' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_array_append(v, "$", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5,3]')] + >>> df.select(to_json(variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5]')] + >>> df.select(to_json(variant_array_append(v, df.path, lit(9))).alias("r")).collect() + [Row(r='[[1,2,9],5]')] + >>> nested = variant_array_append(v, "$", parse_json(lit('[4, 5]'))) + >>> df.select(to_json(nested).alias("r")).collect() + [Row(r='[[1,2],5,[4,5]]')] + >>> df.select(variant_array_append(v, "$", lit(None)).alias("r")).collect() + [Row(r=None)] """ - return _invoke_function_over_columns("bool_and", col) + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "variant_array_append", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) @_try_remote_functions -def some(col: "ColumnOrName") -> Column: +def try_variant_array_append( + v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" +) -> Column: """ - Aggregate function: returns true if at least one value of `col` is true. + Appends a value to the array in a variant at the given JSONPath location. Returns the variant + unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an + incompatible type, the target is not an array, or if any argument is NULL. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to check if at least one value is true. - A column that evaluates to a boolean. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + path : :class:`~pyspark.sql.Column` or str + the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` + supplies the path at runtime. A valid path should start with `$` and is followed by zero or + more segments like `[123]`, `.name`, `['name']`, or `["name"]`. + value : :class:`~pyspark.sql.Column` or str + the value to append. Any expression castable to variant. Returns ------- :class:`~pyspark.sql.Column` - true if at least one value of `col` is true, false otherwise. - - See Also - -------- - :meth:`pyspark.sql.functions.every` + a variant column with `value` appended to the array at `path` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [True], [True]], ["flag"] - ... ).select(sf.some("flag")).show() - +----------+ - |some(flag)| - +----------+ - | true| - +----------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[True], [False], [True]], ["flag"] - ... ).select(sf.some("flag")).show() - +----------+ - |some(flag)| - +----------+ - | true| - +----------+ - - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [[False], [False], [False]], ["flag"] - ... ).select(sf.some("flag")).show() - +----------+ - |some(flag)| - +----------+ - | false| - +----------+ + >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_array_append + >>> df = spark.createDataFrame([{ + ... 'json': '''[[1, 2], 5]''', + ... 'path': '$[0]' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(try_variant_array_append(v, "$", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5,3]')] + >>> df.select(to_json(try_variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() + [Row(r='[[1,2],5]')] + >>> df.select(to_json(try_variant_array_append(v, df.path, lit(9))).alias("r")).collect() + [Row(r='[[1,2,9],5]')] + >>> df.select(to_json(try_variant_array_append(v, "$[1]", lit(9))).alias("r")).collect() + [Row(r=None)] + >>> df.select(try_variant_array_append(v, "$", lit(None)).alias("r")).collect() + [Row(r=None)] """ - return _invoke_function_over_columns("some", col) + from pyspark.sql.classic.column import _to_java_column + + path_col = path if isinstance(path, Column) else lit(path) + return _invoke_function( + "try_variant_array_append", + _to_java_column(v), + _to_java_column(path_col), + _to_java_column(value), + ) @_try_remote_functions -def bool_or(col: "ColumnOrName") -> Column: +def variant_strip_nulls(v: "ColumnOrName", include_arrays: bool = True) -> Column: """ - Aggregate function: returns true if at least one value of `col` is true. + Recursively removes object fields and array elements whose value is a variant null, unless + `include_arrays` is False, in which case null array elements are kept. Returns NULL if any + argument is NULL. - .. versionadded:: 3.5.0 + .. versionadded:: 4.3.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to check if at least one value is true. - A column that evaluates to a boolean. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + include_arrays : bool, optional + whether null elements are also removed from arrays (default True). Returns ------- :class:`~pyspark.sql.Column` - true if at least one value of `col` is true, false otherwise. - - See Also - -------- - :meth:`pyspark.sql.functions.bool_and` + a variant column with variant null fields/elements removed Examples -------- - >>> df = spark.createDataFrame([[True], [True], [True]], ["flag"]) - >>> df.select(bool_or("flag")).show() - +-------------+ - |bool_or(flag)| - +-------------+ - | true| - +-------------+ - >>> df = spark.createDataFrame([[True], [False], [True]], ["flag"]) - >>> df.select(bool_or("flag")).show() - +-------------+ - |bool_or(flag)| - +-------------+ - | true| - +-------------+ - >>> df = spark.createDataFrame([[False], [False], [False]], ["flag"]) - >>> df.select(bool_or("flag")).show() - +-------------+ - |bool_or(flag)| - +-------------+ - | false| - +-------------+ + >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_strip_nulls + >>> df = spark.createDataFrame([{ + ... 'json': '''{ "a" : 1, "b" : null, "c" : [1, null], "d" : { "e" : null, "f" : 4 } }''' + ... }]) + >>> v = parse_json(df.json) + >>> df.select(to_json(variant_strip_nulls(v)).alias("r")).collect() + [Row(r='{"a":1,"c":[1],"d":{"f":4}}')] + >>> df.select(to_json(variant_strip_nulls(v, False)).alias("r")).collect() + [Row(r='{"a":1,"c":[1,null],"d":{"f":4}}')] + >>> df.select(variant_strip_nulls(lit(None)).alias("r")).collect() + [Row(r=None)] + >>> df2 = spark.createDataFrame([{'json': '{"a": null}'}, {'json': 'null'}]) + >>> v2 = parse_json(df2.json) + >>> df2.select(to_json(variant_strip_nulls(v2)).alias("r")).collect() + [Row(r='{}'), Row(r='null')] """ - return _invoke_function_over_columns("bool_or", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function( + "variant_strip_nulls", _to_java_column(v), _enum_to_value(include_arrays) + ) @_try_remote_functions -def bit_and(col: "ColumnOrName") -> Column: +def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: """ - Aggregate function: returns the bitwise AND of all non-null input values, or null if none. + Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to + `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + a column containing the extraction path strings or a string representing the extraction + path. A valid path should start with `$` and is followed by zero or more segments like + A column that evaluates to a string. + `[123]`, `.name`, `['name']`, or `["name"]`. + targetType : str + A DDL-formatted type string. Must be a constant. + the target data type to cast into, in a DDL-formatted string Returns ------- :class:`~pyspark.sql.Column` - the bitwise AND of all non-null input values, or null if none. - - See Also - -------- - :meth:`pyspark.sql.functions.bit_or` - :meth:`pyspark.sql.functions.bit_xor` + a column of `targetType` representing the extracted result + Returns a column of the type given by `targetType`. Examples -------- - Example 1: Bitwise AND with all non-null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | 0| - +----------+ - - Example 2: Bitwise AND with null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | 0| - +----------+ - - Example 3: Bitwise AND with all null values - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([StructField("c", IntegerType(), True)]) - >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | NULL| - +----------+ - - Example 4: Bitwise AND with single input value - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[5]], ["c"]) - >>> df.select(sf.bit_and("c")).show() - +----------+ - |bit_and(c)| - +----------+ - | 5| - +----------+ + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) + >>> df.select(variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() + [Row(r=1)] + >>> df.select(variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() + [Row(r=None)] + >>> df.select(variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() + [Row(r=1)] """ - return _invoke_function_over_columns("bit_and", col) + from pyspark.sql.classic.column import _to_java_column + + assert isinstance(path, (Column, str)) + if isinstance(path, str): + return _invoke_function( + "variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) + ) + else: + return _invoke_function( + "variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) + ) @_try_remote_functions -def bit_or(col: "ColumnOrName") -> Column: +def try_variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: """ - Aggregate function: returns the bitwise OR of all non-null input values, or null if none. + Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to + `targetType`. Returns null if the path does not exist or the cast fails. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. + path : :class:`~pyspark.sql.Column` or str + a column containing the extraction path strings or a string representing the extraction + path. A valid path should start with `$` and is followed by zero or more segments like + A column that evaluates to a string. + `[123]`, `.name`, `['name']`, or `["name"]`. + targetType : str + A DDL-formatted type string. Must be a constant. + the target data type to cast into, in a DDL-formatted string Returns ------- :class:`~pyspark.sql.Column` - the bitwise OR of all non-null input values, or null if none. - - See Also - -------- - :meth:`pyspark.sql.functions.bit_and` - :meth:`pyspark.sql.functions.bit_xor` + a column of `targetType` representing the extracted result + Returns a column of the type given by `targetType`. Examples -------- - Example 1: Bitwise OR with all non-null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | 3| - +---------+ - - Example 2: Bitwise OR with some null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | 3| - +---------+ - - Example 3: Bitwise OR with all null values - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([StructField("c", IntegerType(), True)]) - >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | NULL| - +---------+ - - Example 4: Bitwise OR with single input value - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[5]], ["c"]) - >>> df.select(sf.bit_or("c")).show() - +---------+ - |bit_or(c)| - +---------+ - | 5| - +---------+ + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) + >>> df.select(try_variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() + [Row(r=1)] + >>> df.select(try_variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() + [Row(r=None)] + >>> df.select(try_variant_get(parse_json(df.json), "$.a", "binary").alias("r")).collect() + [Row(r=None)] + >>> df.select(try_variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() + [Row(r=1)] """ - return _invoke_function_over_columns("bit_or", col) + from pyspark.sql.classic.column import _to_java_column + + if isinstance(path, str): + return _invoke_function( + "try_variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) + ) + else: + return _invoke_function( + "try_variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) + ) @_try_remote_functions -def bit_xor(col: "ColumnOrName") -> Column: +def schema_of_variant(v: "ColumnOrName") -> Column: """ - Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. + Returns schema in the SQL format of a variant. - .. versionadded:: 3.5.0 + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to an integral. + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. Returns ------- :class:`~pyspark.sql.Column` - the bitwise XOR of all non-null input values, or null if none. - - See Also - -------- - :meth:`pyspark.sql.functions.bit_and` - :meth:`pyspark.sql.functions.bit_or` + a string column representing the variant schema + Returns a column that evaluates to a string. Examples -------- - Example 1: Bitwise XOR with all non-null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | 2| - +----------+ - - Example 2: Bitwise XOR with some null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | 3| - +----------+ - - Example 3: Bitwise XOR with all null values - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import IntegerType, StructType, StructField - >>> schema = StructType([StructField("c", IntegerType(), True)]) - >>> df = spark.createDataFrame([[None],[None],[None]], schema=schema) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | NULL| - +----------+ - - Example 4: Bitwise XOR with single input value - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[5]], ["c"]) - >>> df.select(sf.bit_xor("c")).show() - +----------+ - |bit_xor(c)| - +----------+ - | 5| - +----------+ + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(schema_of_variant(parse_json(df.json)).alias("r")).collect() + [Row(r='OBJECT')] """ - return _invoke_function_over_columns("bit_xor", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("schema_of_variant", _to_java_column(v)) @_try_remote_functions -def skewness(col: "ColumnOrName") -> Column: +def schema_of_variant_agg(v: "ColumnOrName") -> Column: """ - Aggregate function: returns the skewness of the values in a group. - - .. versionadded:: 1.6.0 + Returns the merged schema in the SQL format of a variant column. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. - - See Also - -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.kurtosis` + v : :class:`~pyspark.sql.Column` or str + a variant column or column name + A column that evaluates to a variant. Returns ------- :class:`~pyspark.sql.Column` - skewness of given column. + a string column representing the variant schema + Returns a column that evaluates to a string. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.skewness(df.c)).show() - +------------------+ - | skewness(c)| - +------------------+ - |0.7071067811865...| - +------------------+ + >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) + >>> df.select(schema_of_variant_agg(parse_json(df.json)).alias("r")).collect() + [Row(r='OBJECT')] """ - return _invoke_function_over_columns("skewness", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("schema_of_variant_agg", _to_java_column(v)) @_try_remote_functions -def kurtosis(col: "ColumnOrName") -> Column: +def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: """ - Aggregate function: returns the kurtosis of the values in a group. + Converts a column containing a :class:`StructType`, :class:`ArrayType`, :class:`MapType` + or a :class:`VariantType` into a JSON string. Throws an exception, in the case of an unsupported type. - .. versionadded:: 1.6.0 + .. versionadded:: 2.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - A column that evaluates to a numeric. - - Returns + col : :class:`~pyspark.sql.Column` or str + name of column containing a struct, an array, a map, or a variant object. + A column that evaluates to a struct, array, map, or variant. + options : dict, optional + options to control converting. accepts the same options as the JSON datasource. + See `Data Source Option `_ + for the version you use. + Additionally the function supports the `pretty` option which enables + A dict of options. Each key and value is a string. + pretty JSON generation. + + .. # noqa + + Returns ------- :class:`~pyspark.sql.Column` - kurtosis of given column. - - See Also - -------- - :meth:`pyspark.sql.functions.std` - :meth:`pyspark.sql.functions.stddev` - :meth:`pyspark.sql.functions.variance` - :meth:`pyspark.sql.functions.skewness` + JSON object as string column. + Returns a column that evaluates to a string. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.select(sf.kurtosis(df.c)).show() - +-----------+ - |kurtosis(c)| - +-----------+ - | -1.5| - +-----------+ + Example 1: Converting a StructType column to JSON + + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql import Row + >>> data = [(1, Row(age=2, name='Alice'))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +------------------------+ + |json | + +------------------------+ + |{"age":2,"name":"Alice"}| + +------------------------+ + + Example 2: Converting an ArrayType column to JSON + + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql import Row + >>> data = [(1, [Row(age=2, name='Alice'), Row(age=3, name='Bob')])] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +-------------------------------------------------+ + |json | + +-------------------------------------------------+ + |[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]| + +-------------------------------------------------+ + + Example 3: Converting a MapType column to JSON + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, {"name": "Alice"})], ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +----------------+ + |json | + +----------------+ + |{"name":"Alice"}| + +----------------+ + + Example 4: Converting a VariantType column to JSON + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, '{"name": "Alice"}')], ("key", "value")) + >>> df.select(sf.to_json(sf.parse_json(df.value)).alias("json")).show(truncate=False) + +----------------+ + |json | + +----------------+ + |{"name":"Alice"}| + +----------------+ + + Example 5: Converting a nested MapType column to JSON + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, [{"name": "Alice"}, {"name": "Bob"}])], ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +---------------------------------+ + |json | + +---------------------------------+ + |[{"name":"Alice"},{"name":"Bob"}]| + +---------------------------------+ + + Example 6: Converting a simple ArrayType column to JSON + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(1, ["Alice", "Bob"])], ("key", "value")) + >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) + +---------------+ + |json | + +---------------+ + |["Alice","Bob"]| + +---------------+ + + Example 7: Converting to JSON with specified options + + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT (DATE('2022-02-22'), 1) AS date") + >>> json1 = sf.to_json(df.date) + >>> json2 = sf.to_json(df.date, {"dateFormat": "yyyy/MM/dd"}) + >>> df.select("date", json1, json2).show(truncate=False) + +---------------+------------------------------+------------------------------+ + |date |to_json(date) |to_json(date) | + +---------------+------------------------------+------------------------------+ + |{2022-02-22, 1}|{"col1":"2022-02-22","col2":1}|{"col1":"2022/02/22","col2":1}| + +---------------+------------------------------+------------------------------+ """ - return _invoke_function_over_columns("kurtosis", col) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("to_json", _to_java_column(col), _options_to_str(options)) @_try_remote_functions -def collect_list(col: "ColumnOrName") -> Column: +def schema_of_json(json: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: """ - Aggregate function: Collects the values from a column into a list, - maintaining duplicates, and returns this list of objects. + Parses a JSON string and infers its schema in DDL format. - .. versionadded:: 1.6.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the function is computed. + json : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + a JSON string or a foldable string column containing a JSON string. + options : dict, optional + options to control parsing. accepts the same options as the JSON datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. - See Also - -------- - :meth:`pyspark.sql.functions.array_agg` - :meth:`pyspark.sql.functions.collect_set` + .. # noqa + + .. versionchanged:: 3.0.0 + It accepts `options` parameter to control schema inferring. Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing a list of collected values, with duplicate values included. - - Notes - ----- - The function is non-deterministic as the order of collected results depends - on the order of the rows, which possibly becomes non-deterministic after shuffle operations. + a string representation of a :class:`StructType` parsed from given JSON. + Returns a column that evaluates to a string. Examples -------- - Example 1: Collect values from a DataFrame and sort the result in ascending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) - >>> df.select(sf.sort_array(sf.collect_list('value')).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [1, 2, 2]| - +-----------+ - - Example 2: Collect values from a DataFrame and sort the result in descending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) - >>> df.select(sf.sort_array(sf.collect_list('age'), asc=False).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [5, 5, 2]| - +-----------+ + >>> import pyspark.sql.functions as sf + >>> parsed1 = sf.schema_of_json(sf.lit('{"a": 0}')) + >>> parsed2 = sf.schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'}) + >>> spark.range(1).select(parsed1, parsed2).show() + +------------------------+----------------------+ + |schema_of_json({"a": 0})|schema_of_json({a: 1})| + +------------------------+----------------------+ + | STRUCT| STRUCT| + +------------------------+----------------------+ + """ + from pyspark.sql.classic.column import _to_java_column - Example 3: Collect values from a DataFrame with multiple columns and sort the result + json = _enum_to_value(json) + if not isinstance(json, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "json", + "arg_type": type(json).__name__, + }, + ) - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) - >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_list('id')).alias('sorted_list')) - >>> df.orderBy(sf.desc("name")).show() - +----+-----------+ - |name|sorted_list| - +----+-----------+ - |John| [1, 2]| - | Ana| [3]| - +----+-----------+ - """ - return _invoke_function_over_columns("collect_list", col) + return _invoke_function("schema_of_json", _to_java_column(lit(json)), _options_to_str(options)) @_try_remote_functions -def array_agg(col: "ColumnOrName") -> Column: +def json_array_length(col: "ColumnOrName") -> Column: """ - Aggregate function: returns a list of objects with duplicates. + Returns the number of elements in the outermost JSON array. `NULL` is returned in case of + any other valid JSON string, `NULL` or an invalid JSON. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + col: :class:`~pyspark.sql.Column` or str target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - list of objects with duplicates. - - See Also - -------- - :meth:`pyspark.sql.functions.collect_list` - :meth:`pyspark.sql.functions.collect_set` + length of json array. + Returns a column that evaluates to an integer. Examples -------- - Example 1: Using array_agg function on an int column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[1],[2]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [1, 1, 2]| - +-----------+ - - Example 2: Using array_agg function on a string column - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([["apple"],["apple"],["banana"]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show(truncate=False) - +----------------------+ - |sorted_list | - +----------------------+ - |[apple, apple, banana]| - +----------------------+ - - Example 3: Using array_agg function on a column with null values - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],[None],[2]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() - +-----------+ - |sorted_list| - +-----------+ - | [1, 2]| - +-----------+ - - Example 4: Using array_agg function on a column with different data types - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([[1],["apple"],[2]], ["c"]) - >>> df.agg(sf.sort_array(sf.array_agg('c')).alias('sorted_list')).show() - +-------------+ - | sorted_list| - +-------------+ - |[1, 2, apple]| - +-------------+ + >>> df = spark.createDataFrame([(None,), ('[1, 2, 3]',), ('[]',)], ['data']) + >>> df.select(json_array_length(df.data).alias('r')).collect() + [Row(r=None), Row(r=3), Row(r=0)] """ - return _invoke_function_over_columns("array_agg", col) + return _invoke_function_over_columns("json_array_length", col) @_try_remote_functions -def collect_set(col: "ColumnOrName") -> Column: +def json_object_keys(col: "ColumnOrName") -> Column: """ - Aggregate function: Collects the values from a column into a set, - eliminating duplicates, and returns this set of objects. - - .. versionadded:: 1.6.0 + Returns all the keys of the outermost JSON object as an array. If a valid JSON object is + given, all the keys of the outermost object will be returned as an array. If it is any + other valid JSON string, an invalid JSON string or an empty string, the function returns null. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target column on which the function is computed. + col: :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing a set of collected values, duplicates excluded. - - See Also - -------- - :meth:`pyspark.sql.functions.array_agg` - :meth:`pyspark.sql.functions.collect_list` - - Notes - ----- - This function is non-deterministic as the order of collected results depends - on the order of the rows, which may be non-deterministic after any shuffle operations. + all the keys of the outermost JSON object. + Returns a column that evaluates to an array. Examples -------- - Example 1: Collect values from a DataFrame and sort the result in ascending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (2,)], ('value',)) - >>> df.select(sf.sort_array(sf.collect_set('value')).alias('sorted_set')).show() - +----------+ - |sorted_set| - +----------+ - | [1, 2]| - +----------+ - - Example 2: Collect values from a DataFrame and sort the result in descending order - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(2,), (5,), (5,)], ('age',)) - >>> df.select(sf.sort_array(sf.collect_set('age'), asc=False).alias('sorted_set')).show() - +----------+ - |sorted_set| - +----------+ - | [5, 2]| - +----------+ - - Example 3: Collect values from a DataFrame with multiple columns and sort the result - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, "John"), (2, "John"), (3, "Ana")], ("id", "name")) - >>> df = df.groupBy("name").agg(sf.sort_array(sf.collect_set('id')).alias('sorted_set')) - >>> df.orderBy(sf.desc("name")).show() - +----+----------+ - |name|sorted_set| - +----+----------+ - |John| [1, 2]| - | Ana| [3]| - +----+----------+ + >>> df = spark.createDataFrame([(None,), ('{}',), ('{"key1":1, "key2":2}',)], ['data']) + >>> df.select(json_object_keys(df.data).alias('r')).collect() + [Row(r=None), Row(r=[]), Row(r=['key1', 'key2'])] """ - return _invoke_function_over_columns("collect_set", col) + return _invoke_function_over_columns("json_object_keys", col) @_try_remote_functions -def collect_union(col: "ColumnOrName") -> Column: +def json_typeof(col: "ColumnOrName") -> Column: """ - Aggregate function: given an array-typed column, collects the distinct union of the - elements of the arrays across rows and returns it as an array. - - The aggregation buffer holds only the distinct elements, so its size is bounded by the - element universe rather than by the number of input rows. Null elements are dropped by - default (``IGNORE NULLS``), matching :func:`collect_set`. With ``RESPECT NULLS`` a single - null element is kept, in which case this is equivalent to - ``array_distinct(flatten(collect_list(col)))``. The ``RESPECT NULLS`` clause is only - available through SQL, e.g. ``expr("collect_union(col) RESPECT NULLS")``. + Returns the type of the outermost JSON value as a string: one of 'object', 'array', + 'string', 'number', 'boolean', or 'null'. Returns null if the input is not a valid JSON + string or is an empty string. - .. versionadded:: 4.3.0 + .. versionadded:: 4.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The target array column on which the function is computed. + col: :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing the distinct union of the array elements. + the type of the outermost JSON value. + Returns a column that evaluates to a string. See Also -------- - :meth:`pyspark.sql.functions.collect_set` - :meth:`pyspark.sql.functions.collect_list` - :meth:`pyspark.sql.functions.array_distinct` - :meth:`pyspark.sql.functions.flatten` - - Notes - ----- - This function is non-deterministic as the order of collected results depends - on the order of the rows, which may be non-deterministic after any shuffle operations. + :meth:`pyspark.sql.functions.json_object_keys` + :meth:`pyspark.sql.functions.get_json_object` + :meth:`pyspark.sql.functions.json_array_length` Examples -------- - Example 1: Union the elements of array columns across rows - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [([1, 2],), ([2, 3],), ([1],)], ('value',)) - >>> df.select(sf.sort_array(sf.collect_union('value')).alias('u')).show() - +---------+ - | u| - +---------+ - |[1, 2, 3]| - +---------+ - - Example 2: Union per group - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("a", [1, 2]), ("a", [2, 3]), ("b", [4])], ("k", "value")) - >>> df = df.groupBy("k").agg(sf.sort_array(sf.collect_union('value')).alias('u')) - >>> df.orderBy("k").show() - +---+---------+ - | k| u| - +---+---------+ - | a|[1, 2, 3]| - | b| [4]| - +---+---------+ - """ - return _invoke_function_over_columns("collect_union", col) - - -@_try_remote_functions -def approxCountDistinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: - """ - This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates - the approximate distinct count of elements in a specified column or a group of columns. - - .. versionadded:: 1.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. deprecated:: 2.1.0 - Use :func:`approx_count_distinct` instead. + >>> df = spark.createDataFrame([('{"a": 1}',), ('[1, 2, 3]',), ('123',), ('',)], ['data']) + >>> df.select(json_typeof(df.data).alias('r')).collect() + [Row(r='object'), Row(r='array'), Row(r='number'), Row(r=None)] """ - warnings.warn("Deprecated in 2.1, use approx_count_distinct instead.", FutureWarning) - return approx_count_distinct(col, rsd) + return _invoke_function_over_columns("json_typeof", col) +# TODO: Fix and add an example for StructType with Spark Connect +# e.g., StructType([StructField("a", IntegerType())]) @_try_remote_functions -def approx_count_distinct(col: "ColumnOrName", rsd: Optional[float] = None) -> Column: +def from_xml( + col: "ColumnOrName", + schema: Union[StructType, Column, str], + options: Optional[Mapping[str, str]] = None, +) -> Column: """ - This aggregate function returns a new :class:`~pyspark.sql.Column`, which estimates - the approximate distinct count of elements in a specified column or a group of columns. - - .. versionadded:: 2.1.0 + Parses a column containing a XML string to a row with + the specified schema. Returns `null`, in the case of an unparsable string. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The label of the column to count distinct values in. - rsd : float, optional - The maximum allowed relative standard deviation (default = 0.05). - If rsd < 0.01, it would be more efficient to use :func:`count_distinct`. + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + a column or column name in XML format + schema : :class:`StructType`, :class:`~pyspark.sql.Column` or str + a StructType, Column or Python string literal with a DDL-formatted string + A column that evaluates to a string, or a DDL-formatted type string, or a DataType. + to use when parsing the Xml column + options : dict, optional + options to control parsing. accepts the same options as the Xml datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - A new Column object representing the approximate unique count. - - See Also - -------- - :meth:`pyspark.sql.functions.count_distinct` + a new column of complex type from given XML object. + Returns a column that evaluates to a struct. Examples -------- - Example 1: Counting distinct values in a single column DataFrame representing integers + Example 1: Parsing XML with a DDL-formatted string schema - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "int") - >>> df.agg(sf.approx_count_distinct("value")).show() - +----------------------------+ - |approx_count_distinct(value)| - +----------------------------+ - | 3| - +----------------------------+ + >>> import pyspark.sql.functions as sf + >>> data = [(1, '''

1

''')] + >>> df = spark.createDataFrame(data, ("key", "value")) + ... # Define the schema using a DDL-formatted string + >>> schema = "STRUCT" + ... # Parse the XML column using the DDL-formatted schema + >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() + [Row(xml=Row(a=1))] - Example 2: Counting distinct values in a single column DataFrame representing strings + Example 2: Parsing XML with a :class:`StructType` schema - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("apple",), ("orange",), ("apple",), ("banana",)], ['fruit']) - >>> df.agg(sf.approx_count_distinct("fruit")).show() - +----------------------------+ - |approx_count_distinct(fruit)| - +----------------------------+ - | 3| - +----------------------------+ + >>> import pyspark.sql.functions as sf + >>> from pyspark.sql.types import StructType, LongType + >>> data = [(1, '''

1

''')] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> schema = StructType().add("a", LongType()) + >>> df.select(sf.from_xml(df.value, schema)).show() + +---------------+ + |from_xml(value)| + +---------------+ + | {1}| + +---------------+ - Example 3: Counting distinct values in a DataFrame with multiple columns + Example 3: Parsing XML with :class:`ArrayType` in schema - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("Alice", 1), ("Alice", 2), ("Bob", 3), ("Bob", 3)], ["name", "value"]) - >>> df = df.withColumn("combined", sf.struct("name", "value")) - >>> df.agg(sf.approx_count_distinct(df.combined)).show() - +-------------------------------+ - |approx_count_distinct(combined)| - +-------------------------------+ - | 3| - +-------------------------------+ + >>> import pyspark.sql.functions as sf + >>> data = [(1, '

12

')] + >>> df = spark.createDataFrame(data, ("key", "value")) + ... # Define the schema with an Array type + >>> schema = "STRUCT>" + ... # Parse the XML column using the schema with an Array + >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() + [Row(xml=Row(a=[1, 2]))] - Example 4: Counting distinct values with a specified relative standard deviation + Example 4: Parsing XML using :meth:`pyspark.sql.functions.schema_of_xml` - >>> from pyspark.sql import functions as sf - >>> spark.range(100000).agg( - ... sf.approx_count_distinct("id").alias('with_default_rsd'), - ... sf.approx_count_distinct("id", 0.1).alias('with_rsd_0.1') - ... ).show() - +----------------+------------+ - |with_default_rsd|with_rsd_0.1| - +----------------+------------+ - | 95546| 102065| - +----------------+------------+ + >>> import pyspark.sql.functions as sf + >>> # Sample data with an XML column + ... data = [(1, '

12

')] + >>> df = spark.createDataFrame(data, ("key", "value")) + ... # Generate the schema from an example XML value + >>> schema = sf.schema_of_xml(sf.lit(data[0][1])) + ... # Parse the XML column using the generated schema + >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() + [Row(xml=Row(a=[1, 2]))] + + See Also + -------- + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` """ from pyspark.sql.classic.column import _to_java_column - if rsd is None: - return _invoke_function_over_columns("approx_count_distinct", col) - else: - return _invoke_function("approx_count_distinct", _to_java_column(col), _enum_to_value(rsd)) + if isinstance(schema, StructType): + schema = schema.json() + elif isinstance(schema, Column): + schema = _to_java_column(schema) + elif not isinstance(schema, str): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "StructType, Column or str", + "arg_name": "schema", + "arg_type": type(schema).__name__, + }, + ) + return _invoke_function("from_xml", _to_java_column(col), schema, _options_to_str(options)) @_try_remote_functions -def corr(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for the Pearson Correlation Coefficient for - ``col1`` and ``col2``. - - .. versionadded:: 1.6.0 +def schema_of_xml(xml: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: + """ + Parses a XML string and infers its schema in DDL format. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to calculate correlation. - A column that evaluates to a numeric. - col2 : :class:`~pyspark.sql.Column` or column name - second column to calculate correlation. - A column that evaluates to a numeric. + xml : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + a XML string or a foldable string column containing a XML string. + options : dict, optional + options to control parsing. accepts the same options as the XML datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - Pearson Correlation Coefficient of these two column values. + a string representation of a :class:`StructType` parsed from given XML. + Returns a column that evaluates to a string. Examples -------- + Example 1: Parsing a simple XML with a single element + >>> from pyspark.sql import functions as sf - >>> a = range(20) - >>> b = [2 * x for x in range(20)] - >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) - >>> df.agg(sf.corr("a", df.b)).show() - +----------+ - |corr(a, b)| - +----------+ - | 1.0| - +----------+ + >>> df = spark.range(1) + >>> df.select(sf.schema_of_xml(sf.lit('

1

')).alias("xml")).collect() + [Row(xml='STRUCT')] + + Example 2: Parsing an XML with multiple elements in an array + + >>> from pyspark.sql import functions as sf + >>> df.select(sf.schema_of_xml(sf.lit('

12

')).alias("xml")).collect() + [Row(xml='STRUCT>')] + + Example 3: Parsing XML with options to exclude attributes + + >>> from pyspark.sql import functions as sf + >>> schema = sf.schema_of_xml('

1

', {'excludeAttribute':'true'}) + >>> df.select(schema.alias("xml")).collect() + [Row(xml='STRUCT')] + + Example 4: Parsing XML with complex structure + + >>> from pyspark.sql import functions as sf + >>> df.select( + ... sf.schema_of_xml( + ... sf.lit('Alice30') + ... ).alias("xml") + ... ).collect() + [Row(xml='STRUCT>')] + + Example 5: Parsing XML with nested arrays + + >>> from pyspark.sql import functions as sf + >>> df.select( + ... sf.schema_of_xml( + ... sf.lit('12') + ... ).alias("xml") + ... ).collect() + [Row(xml='STRUCT>>')] + + See Also + -------- + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.to_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` """ - return _invoke_function_over_columns("corr", col1, col2) + from pyspark.sql.classic.column import _to_java_column + xml = _enum_to_value(xml) + if not isinstance(xml, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "xml", + "arg_type": type(xml).__name__, + }, + ) -@_try_remote_functions -def covar_pop(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for the population covariance of ``col1`` and - ``col2``. + return _invoke_function("schema_of_xml", _to_java_column(lit(xml)), _options_to_str(options)) - .. versionadded:: 2.0.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def to_xml(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: + """ + Converts a column containing a :class:`StructType` into a XML string. + Throws an exception, in the case of an unsupported type. + + .. versionadded:: 4.0.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to calculate covariance. - A column that evaluates to a numeric. - col2 : :class:`~pyspark.sql.Column` or column name - second column to calculate covariance. - A column that evaluates to a numeric. + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a struct, array, map, or variant. + name of column containing a struct. + options: dict, optional + options to control converting. accepts the same options as the XML datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - covariance of these two column values. + a XML string converted from given :class:`StructType`. + Returns a column that evaluates to a string. - See Also + Examples -------- - :meth:`pyspark.sql.functions.covar_samp` + >>> from pyspark.sql import Row + >>> data = [(1, Row(age=2, name='Alice'))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(to_xml(df.value, {'rowTag':'person'}).alias("xml")).collect() + [Row(xml='\\n 2\\n Alice\\n')] - Examples + See Also -------- - >>> from pyspark.sql import functions as sf - >>> a = [1] * 10 - >>> b = [1] * 10 - >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) - >>> df.agg(sf.covar_pop("a", df.b)).show() - +---------------+ - |covar_pop(a, b)| - +---------------+ - | 0.0| - +---------------+ + :meth:`pyspark.sql.functions.from_xml` + :meth:`pyspark.sql.functions.schema_of_xml` + :meth:`pyspark.sql.functions.xpath` + :meth:`pyspark.sql.functions.xpath_boolean` + :meth:`pyspark.sql.functions.xpath_double` + :meth:`pyspark.sql.functions.xpath_float` + :meth:`pyspark.sql.functions.xpath_int` + :meth:`pyspark.sql.functions.xpath_long` + :meth:`pyspark.sql.functions.xpath_number` + :meth:`pyspark.sql.functions.xpath_short` + :meth:`pyspark.sql.functions.xpath_string` """ - return _invoke_function_over_columns("covar_pop", col1, col2) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("to_xml", _to_java_column(col), _options_to_str(options)) @_try_remote_functions -def covar_samp(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for the sample covariance of ``col1`` and - ``col2``. +def schema_of_csv(csv: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: + """ + CSV Function: Parses a CSV string and infers its schema in DDL format. - .. versionadded:: 2.0.0 + .. versionadded:: 3.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - first column to calculate covariance. - A column that evaluates to a numeric. - col2 : :class:`~pyspark.sql.Column` or column name - second column to calculate covariance. - A column that evaluates to a numeric. + csv : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + A CSV string or a foldable string column containing a CSV string. + options : dict, optional + Options to control parsing. Accepts the same options as the CSV datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - sample covariance of these two column values. - - See Also - -------- - :meth:`pyspark.sql.functions.covar_pop` + A string representation of a :class:`StructType` parsed from the given CSV. + Returns a column that evaluates to a string. Examples -------- + Example 1: Inferring the schema of a CSV string with different data types + >>> from pyspark.sql import functions as sf - >>> a = [1] * 10 - >>> b = [1] * 10 - >>> df = spark.createDataFrame(zip(a, b), ["a", "b"]) - >>> df.agg(sf.covar_samp("a", df.b)).show() - +----------------+ - |covar_samp(a, b)| - +----------------+ - | 0.0| - +----------------+ - """ - return _invoke_function_over_columns("covar_samp", col1, col2) + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('1|a|true'), {'sep':'|'})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv(1|a|true) | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ + Example 2: Inferring the schema of a CSV string with missing values -@_try_remote_functions -def countDistinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: - """Returns a new :class:`~pyspark.sql.Column` for distinct count of ``col`` or ``cols``. + >>> from pyspark.sql import functions as sf + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('1||true'), {'sep':'|'})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv(1||true) | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ - An alias of :func:`count_distinct`, and it is encouraged to use :func:`count_distinct` - directly. + Example 3: Inferring the schema of a CSV string with a different delimiter - .. versionadded:: 1.3.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('1;a;true'), {'sep':';'})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv(1;a;true) | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 4: Inferring the schema of a CSV string with quoted fields - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) - >>> df.select(sf.count_distinct(df.value)).show() - +---------------------+ - |count(DISTINCT value)| - +---------------------+ - | 2| - +---------------------+ - - >>> df.select(sf.countDistinct(df.value)).show() - +---------------------+ - |count(DISTINCT value)| - +---------------------+ - | 2| - +---------------------+ + >>> df = spark.range(1) + >>> df.select(sf.schema_of_csv(sf.lit('"1","a","true"'), {'sep':','})).show(truncate=False) + +-------------------------------------------+ + |schema_of_csv("1","a","true") | + +-------------------------------------------+ + |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| + +-------------------------------------------+ """ - return count_distinct(col, *cols) + from pyspark.sql.classic.column import _to_java_column + + csv = _enum_to_value(csv) + if not isinstance(csv, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "csv", + "arg_type": type(csv).__name__, + }, + ) + + return _invoke_function("schema_of_csv", _to_java_column(lit(csv)), _options_to_str(options)) @_try_remote_functions -def count_distinct(col: "ColumnOrName", *cols: "ColumnOrName") -> Column: - """Returns a new :class:`Column` for distinct count of ``col`` or ``cols``. +def to_csv(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: + """ + CSV Function: Converts a column containing a :class:`StructType` into a CSV string. + Throws an exception, in the case of an unsupported type. - .. versionadded:: 3.2.0 + .. versionadded:: 3.0.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - first column to compute on. - cols : :class:`~pyspark.sql.Column` or column name - other columns to compute on. + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a struct, array, map, or variant. + Name of column containing a struct. + options: dict, optional + Options to control converting. Accepts the same options as the CSV datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - distinct values of these two column values. - - See Also - -------- - :meth:`pyspark.sql.functions.approx_count_distinct` + A CSV string converted from the given :class:`StructType`. + Returns a column that evaluates to a string. Examples -------- - Example 1: Counting distinct values of a single column + Example 1: Converting a simple StructType to a CSV string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (1,), (3,)], ["value"]) - >>> df.select(sf.count_distinct(df.value)).show() - +---------------------+ - |count(DISTINCT value)| - +---------------------+ - | 2| - +---------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> data = [(1, Row(age=2, name='Alice'))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_csv(df.value)).show() + +-------------+ + |to_csv(value)| + +-------------+ + | 2,Alice| + +-------------+ - Example 2: Counting distinct values of multiple columns + Example 2: Converting a complex StructType to a CSV string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) - >>> df.select(sf.count_distinct(df.value1, df.value2)).show() - +------------------------------+ - |count(DISTINCT value1, value2)| - +------------------------------+ - | 2| - +------------------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> data = [(1, Row(age=2, name='Alice', scores=[100, 200, 300]))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_csv(df.value)).show(truncate=False) + +-------------------------+ + |to_csv(value) | + +-------------------------+ + |2,Alice,"[100, 200, 300]"| + +-------------------------+ - Example 3: Counting distinct values with column names as strings + Example 3: Converting a StructType with null values to a CSV string - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1), (1, 2)], ["value1", "value2"]) - >>> df.select(sf.count_distinct("value1", "value2")).show() - +------------------------------+ - |count(DISTINCT value1, value2)| - +------------------------------+ - | 2| - +------------------------------+ + >>> from pyspark.sql import Row, functions as sf + >>> from pyspark.sql.types import StructType, StructField, IntegerType, StringType + >>> data = [(1, Row(age=None, name='Alice'))] + >>> schema = StructType([ + ... StructField("key", IntegerType(), True), + ... StructField("value", StructType([ + ... StructField("age", IntegerType(), True), + ... StructField("name", StringType(), True) + ... ]), True) + ... ]) + >>> df = spark.createDataFrame(data, schema) + >>> df.select(sf.to_csv(df.value)).show() + +-------------+ + |to_csv(value)| + +-------------+ + | ,Alice| + +-------------+ + + Example 4: Converting a StructType with different data types to a CSV string + + >>> from pyspark.sql import Row, functions as sf + >>> data = [(1, Row(age=2, name='Alice', isStudent=True))] + >>> df = spark.createDataFrame(data, ("key", "value")) + >>> df.select(sf.to_csv(df.value)).show() + +-------------+ + |to_csv(value)| + +-------------+ + | 2,Alice,true| + +-------------+ """ - from pyspark.sql.classic.column import _to_java_column, _to_seq + from pyspark.sql.classic.column import _to_java_column - sc = _get_active_spark_context() - return _invoke_function( - "count_distinct", _to_java_column(col), _to_seq(sc, cols, _to_java_column) - ) + return _invoke_function("to_csv", _to_java_column(col), _options_to_str(options)) @_try_remote_functions -def first(col: "ColumnOrName", ignorenulls: bool = False) -> Column: - """Aggregate function: returns the first value in a group. - - The function by default returns the first values it sees. It will return the first non-null - value it sees when ignoreNulls is set to true. If all values are null, then null is returned. +def size(col: "ColumnOrName") -> Column: + """ + Collection function: returns the length of the array or map stored in the column. - .. versionadded:: 1.3.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. - Notes - ----- - The function is non-deterministic because its results depends on the order of the - rows which may be non-deterministic after a shuffle. - Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to fetch first value for. - A column of any type. - ignorenulls : bool - if first value is null then look for first non-null value. ``False`` by default. - A column that evaluates to a boolean. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - first value of the group. + length of the array/map. + Returns a column that evaluates to an integer. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) - >>> df = df.orderBy(df.age) - >>> df.groupby("name").agg(sf.first("age")).orderBy("name").show() - +-----+----------+ - | name|first(age)| - +-----+----------+ - |Alice| NULL| - | Bob| 5| - +-----+----------+ - - To ignore any null values, set ``ignorenulls`` to `True` - - >>> df.groupby("name").agg(sf.first("age", ignorenulls=True)).orderBy("name").show() - +-----+----------+ - | name|first(age)| - +-----+----------+ - |Alice| 2| - | Bob| 5| - +-----+----------+ - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("first", _to_java_column(col), _enum_to_value(ignorenulls)) + >>> df = spark.createDataFrame([([1, 2, 3],),([1],),([],)], ['data']) + >>> df.select(size(df.data)).collect() + [Row(size(data)=3), Row(size(data)=1), Row(size(data)=0)] + """ + return _invoke_function_over_columns("size", col) @_try_remote_functions -def grouping(col: "ColumnOrName") -> Column: +def array_min(col: "ColumnOrName") -> Column: """ - Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated - or not, returns 1 for aggregated or 0 for not aggregated in the result set. + Array function: returns the minimum value of the array. - .. versionadded:: 2.0.0 + .. versionadded:: 2.4.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to check if it's aggregated. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - returns 1 for aggregated or 0 for not aggregated in the result set. + A new column that contains the minimum value of each array. + Returns a column of the element type of the input array. + + See Also + -------- + :meth:`pyspark.sql.functions.array_max` + :meth:`pyspark.sql.functions.array_sort` + :meth:`pyspark.sql.functions.sort_array` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5)], ("name", "age")) - >>> df.cube("name").agg(sf.grouping("name"), sf.sum("age")).orderBy("name").show() - +-----+--------------+--------+ - | name|grouping(name)|sum(age)| - +-----+--------------+--------+ - | NULL| 1| 7| - |Alice| 0| 2| - | Bob| 0| 5| - +-----+--------------+--------+ - """ - return _invoke_function_over_columns("grouping", col) + Example 1: Basic usage with integer array + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | 1| + | -1| + +---------------+ -@_try_remote_functions -def grouping_id(*cols: "ColumnOrName") -> Column: - """ - Aggregate function: returns the level of grouping, equals to + Example 2: Usage with string array - (grouping(c1) << (n-1)) + (grouping(c2) << (n-2)) + ... + grouping(cn) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | apple| + +---------------+ - .. versionadded:: 2.0.0 + Example 3: Usage with mixed type array - .. versionchanged:: 3.4.0 - Supports Spark Connect. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | 1| + +---------------+ - Notes - ----- - The list of columns should match with grouping columns exactly, or empty (means all - the grouping columns). + Example 4: Usage with array of arrays - Parameters - ---------- - cols : :class:`~pyspark.sql.Column` or column name - columns to check for. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | [2, 1]| + +---------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - returns level of the grouping it relates to. + Example 5: Usage with empty array - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(1, "a", "a"), (3, "a", "a"), (4, "b", "c")], ["c1", "c2", "c3"]) - >>> df.cube("c2", "c3").agg(sf.grouping_id(), sf.sum("c1")).orderBy("c2", "c3").show() - +----+----+-------------+-------+ - | c2| c3|grouping_id()|sum(c1)| - +----+----+-------------+-------+ - |NULL|NULL| 3| 8| - |NULL| a| 2| 4| - |NULL| c| 2| 4| - | a|NULL| 1| 4| - | a| a| 0| 4| - | b|NULL| 1| 4| - | b| c| 0| 4| - +----+----+-------------+-------+ + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_min(df.data)).show() + +---------------+ + |array_min(data)| + +---------------+ + | NULL| + +---------------+ """ - return _invoke_function_over_seq_of_columns("grouping_id", cols) + return _invoke_function_over_columns("array_min", col) @_try_remote_functions -def count_min_sketch( - col: "ColumnOrName", - eps: Union[Column, float], - confidence: Union[Column, float], - seed: Optional[Union[Column, int]] = None, -) -> Column: +def array_max(col: "ColumnOrName") -> Column: """ - Returns a count-min sketch of a column with the given esp, confidence and seed. - The result is an array of bytes, which can be deserialized to a `CountMinSketch` before usage. - Count-min sketch is a probabilistic data structure used for cardinality estimation - using sub-linear space. + Array function: returns the maximum value of the array. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to compute on. - eps : :class:`~pyspark.sql.Column` or float - relative error, must be positive - - .. versionchanged:: 4.0.0 - `eps` now accepts float value. - - confidence : :class:`~pyspark.sql.Column` or float - confidence, must be positive and less than 1.0 - - .. versionchanged:: 4.0.0 - `confidence` now accepts float value. - - seed : :class:`~pyspark.sql.Column` or int, optional - random seed - - .. versionchanged:: 4.0.0 - `seed` now accepts int value. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - count-min sketch of the column + A new column that contains the maximum value of each array. + Returns a column of the element type of the input array. + + See Also + -------- + :meth:`pyspark.sql.functions.array_min` + :meth:`pyspark.sql.functions.array_sort` + :meth:`pyspark.sql.functions.sort_array` Examples -------- - Example 1: Using columns as arguments + Example 1: Basic usage with integer array >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch(sf.col("id"), sf.lit(3.0), sf.lit(0.1), sf.lit(1))) - ... ).show(truncate=False) - +------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 3.0, 0.1, 1)) | - +------------------------------------------------------------------------+ - |0000000100000000000000640000000100000001000000005D8D6AB90000000000000064| - +------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([([2, 1, 3],), ([None, 10, -1],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | 3| + | 10| + +---------------+ - Example 2: Using numbers as arguments + Example 2: Usage with string array >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch("id", 1.0, 0.3, 2)) - ... ).show(truncate=False) - +----------------------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 1.0, 0.3, 2)) | - +----------------------------------------------------------------------------------------+ - |0000000100000000000000640000000100000002000000005D96391C00000000000000320000000000000032| - +----------------------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | cherry| + +---------------+ - Example 3: Using a long seed + Example 3: Usage with mixed type array >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.2, 1111111111111111111)) - ... ).show(truncate=False) - +----------------------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 1.5, 0.2, 1111111111111111111)) | - +----------------------------------------------------------------------------------------+ - |00000001000000000000006400000001000000020000000044078BA100000000000000320000000000000032| - +----------------------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | cherry| + +---------------+ - Example 4: Using a random seed + Example 4: Usage with array of arrays >>> from pyspark.sql import functions as sf - >>> spark.range(100).select( - ... sf.hex(sf.count_min_sketch("id", sf.lit(1.5), 0.6)) - ... ).show(truncate=False) # doctest: +SKIP - +----------------------------------------------------------------------------------------------------------------------------------------+ - |hex(count_min_sketch(id, 1.5, 0.6, 2120704260)) | - +----------------------------------------------------------------------------------------------------------------------------------------+ - |0000000100000000000000640000000200000002000000005ADECCEE00000000153EBE090000000000000033000000000000003100000000000000320000000000000032| - +----------------------------------------------------------------------------------------------------------------------------------------+ - """ - _eps = lit(eps) - _conf = lit(confidence) - if seed is None: - return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf) - else: - return _invoke_function_over_columns("count_min_sketch", col, _eps, _conf, lit(seed)) - + >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | [3, 4]| + +---------------+ -@_try_remote_functions -def last(col: "ColumnOrName", ignorenulls: bool = False) -> Column: - """Aggregate function: returns the last value in a group. + Example 5: Usage with empty array - The function by default returns the last values it sees. It will return the last non-null - value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_max(df.data)).show() + +---------------+ + |array_max(data)| + +---------------+ + | NULL| + +---------------+ + """ + return _invoke_function_over_columns("array_max", col) - .. versionadded:: 1.3.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def array_size(col: "ColumnOrName") -> Column: + """ + Array function: returns the total number of elements in the array. + The function returns null for null input. - Notes - ----- - The function is non-deterministic because its results depends on the order of the - rows which may be non-deterministic after a shuffle. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - column to fetch last value for. - A column of any type. - ignorenulls : bool - if last value is null then look for non-null value. ``False`` by default. - A column that evaluates to a boolean. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the array. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - last value of the group. + A new column that contains the size of each array. + Returns a column that evaluates to an integer. + + See Also + -------- + :meth:`pyspark.sql.functions.cardinality` + :meth:`pyspark.sql.functions.size` Examples -------- + Example 1: Basic usage with integer array + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Alice", 2), ("Bob", 5), ("Alice", None)], ("name", "age")) - >>> df = df.orderBy(df.age.desc()) - >>> df.groupby("name").agg(sf.last("age")).orderBy("name").show() - +-----+---------+ - | name|last(age)| - +-----+---------+ - |Alice| NULL| - | Bob| 5| - +-----+---------+ + >>> df = spark.createDataFrame([([2, 1, 3],), (None,)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 3| + | NULL| + +----------------+ - To ignore any null values, set ``ignorenulls`` to `True` + Example 2: Usage with string array - >>> df.groupby("name").agg(sf.last("age", ignorenulls=True)).orderBy("name").show() - +-----+---------+ - | name|last(age)| - +-----+---------+ - |Alice| 2| - | Bob| 5| - +-----+---------+ - """ - from pyspark.sql.classic.column import _to_java_column + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 'banana', 'cherry'],)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 3| + +----------------+ - return _invoke_function("last", _to_java_column(col), _enum_to_value(ignorenulls)) + Example 3: Usage with mixed type array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 1, 'cherry'],)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 3| + +----------------+ + + Example 4: Usage with array of arrays + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([[2, 1], [3, 4]],)], ['data']) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 2| + +----------------+ + + Example 5: Usage with empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType(IntegerType()), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.array_size(df.data)).show() + +----------------+ + |array_size(data)| + +----------------+ + | 0| + +----------------+ + """ + return _invoke_function_over_columns("array_size", col) @_try_remote_functions -def percentile( - col: "ColumnOrName", - percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], - frequency: Union[Column, int] = 1, -) -> Column: - """Returns the exact percentile(s) of numeric column `expr` at the given percentage(s) - with value range in [0.0, 1.0]. +def cardinality(col: "ColumnOrName") -> Column: + """ + Collection function: returns the length of the array or map stored in the column. .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats - percentage in decimal (must be between 0.0 and 1.0). - frequency : :class:`~pyspark.sql.Column` or int is a positive numeric literal which - controls frequency. + col : :class:`~pyspark.sql.Column` or str + target column to compute on. + A column that evaluates to an array or map. Returns ------- :class:`~pyspark.sql.Column` - the exact `percentile` of the numeric column. - - See Also - -------- - :meth:`pyspark.sql.functions.median` - :meth:`pyspark.sql.functions.approx_percentile` - :meth:`pyspark.sql.functions.percentile_approx` + length of the array/map. + Returns a column that evaluates to an integer. Examples -------- - >>> from pyspark.sql import functions as sf - >>> key = (sf.col("id") % 3).alias("key") - >>> value = (sf.randn(42) + key * 10).alias("value") - >>> df = spark.range(0, 1000, 1, 1).select(key, value) - >>> df.select( - ... sf.percentile("value", [0.25, 0.5, 0.75], sf.lit(1)) - ... ).show(truncate=False) - +--------------------------------------------------------+ - |percentile(value, array(0.25, 0.5, 0.75), 1) | - +--------------------------------------------------------+ - |[0.7441991494121..., 9.9900713756..., 19.33740203080...]| - +--------------------------------------------------------+ - - >>> df.groupBy("key").agg( - ... sf.percentile("value", sf.lit(0.5), sf.lit(1)) - ... ).sort("key").show() - +---+-------------------------+ - |key|percentile(value, 0.5, 1)| - +---+-------------------------+ - | 0| -0.03449962216667901| - | 1| 9.990389751837329| - | 2| 19.967859769284075| - +---+-------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.createDataFrame( + ... [([1, 2, 3],),([1],),([],)], ['data'] + ... ).select(sf.cardinality("data")).show() + +-----------------+ + |cardinality(data)| + +-----------------+ + | 3| + | 1| + | 0| + +-----------------+ """ - percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) - return _invoke_function_over_columns("percentile", col, percentage, lit(frequency)) + return _invoke_function_over_columns("cardinality", col) @_try_remote_functions -def percentile_approx( - col: "ColumnOrName", - percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], - accuracy: Union[Column, int] = 10000, -) -> Column: - """Returns the approximate `percentile` of the numeric column `col` which is the smallest value - in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` - of `col` values is less than the value or equal to that value. - +def sort_array(col: "ColumnOrName", asc: bool = True) -> Column: + """ + Array function: Sorts the input array in ascending or descending order according + to the natural ordering of the array elements. Null elements will be placed at the beginning + of the returned array in ascending order or at the end of the returned array in descending + order. - .. versionadded:: 3.1.0 + .. versionadded:: 1.5.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column. - percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats - percentage in decimal (must be between 0.0 and 1.0). - When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. - In this case, returns the approximate percentile array of column col - at the given percentage array. - accuracy : :class:`~pyspark.sql.Column` or int - is a positive numeric literal which controls approximation accuracy - at the cost of memory. Higher value of accuracy yields better accuracy, - 1.0/accuracy is the relative error of the approximation. (default: 10000). + col : :class:`~pyspark.sql.Column` or str + Name of the column or expression. + A column that evaluates to an array. + asc : bool, optional + Whether to sort in ascending or descending order. If `asc` is True (default), + then the sorting is in ascending order. If False, then in descending order. + A column that evaluates to a boolean. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - approximate `percentile` of the numeric column. - - See Also - -------- - :meth:`pyspark.sql.functions.median` - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.approx_percentile` + Sorted array. + Returns a column that evaluates to an array. Examples -------- - >>> from pyspark.sql import functions as sf - >>> key = (sf.col("id") % 3).alias("key") - >>> value = (sf.randn(42) + key * 10).alias("value") - >>> df = spark.range(0, 1000, 1, 1).select(key, value) - >>> df.select( - ... sf.percentile_approx("value", [0.25, 0.5, 0.75], 1000000) - ... ).show(truncate=False) - +----------------------------------------------------------+ - |percentile_approx(value, array(0.25, 0.5, 0.75), 1000000) | - +----------------------------------------------------------+ - |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| - +----------------------------------------------------------+ + Example 1: Sorting an array in ascending order - >>> df.groupBy("key").agg( - ... sf.percentile_approx("value", sf.lit(0.5), sf.lit(1000000)) - ... ).sort("key").show() - +---+--------------------------------------+ - |key|percentile_approx(value, 0.5, 1000000)| - +---+--------------------------------------+ - | 0| -0.03519435193070...| - | 1| 9.990389751837...| - | 2| 19.967859769284...| - +---+--------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | [NULL, 1, 2, 3]| + +----------------------+ + + Example 2: Sorting an array in descending order + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([([2, 1, None, 3],)], ['data']) + >>> df.select(sf.sort_array(df.data, asc=False)).show() + +-----------------------+ + |sort_array(data, false)| + +-----------------------+ + | [3, 2, 1, NULL]| + +-----------------------+ + + Example 3: Sorting an array with a single element + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([([1],)], ['data']) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | [1]| + +----------------------+ + + Example 4: Sorting an empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, StructField, StructType + >>> schema = StructType([StructField("data", ArrayType(StringType()), True)]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | []| + +----------------------+ + + Example 5: Sorting an array with null values + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, IntegerType, StructType, StructField + >>> schema = StructType([StructField("data", ArrayType(IntegerType()), True)]) + >>> df = spark.createDataFrame([([None, None, None],)], schema=schema) + >>> df.select(sf.sort_array(df.data)).show() + +----------------------+ + |sort_array(data, true)| + +----------------------+ + | [NULL, NULL, NULL]| + +----------------------+ """ - percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) - return _invoke_function_over_columns("percentile_approx", col, percentage, lit(accuracy)) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("sort_array", _to_java_column(col), _enum_to_value(asc)) @_try_remote_functions -def approx_percentile( - col: "ColumnOrName", - percentage: Union[Column, float, Sequence[float], Tuple[float, ...]], - accuracy: Union[Column, int] = 10000, +def array_sort( + col: "ColumnOrName", comparator: Optional[Callable[[Column, Column], Column]] = None ) -> Column: - """Returns the approximate `percentile` of the numeric column `col` which is the smallest value - in the ordered `col` values (sorted from least to greatest) such that no more than `percentage` - of `col` values is less than the value or equal to that value. + """ + Collection function: sorts the input array in ascending order. The elements of the input array + must be orderable. Null elements will be placed at the end of the returned array. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Can take a `comparator` function. + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input column. - percentage : :class:`~pyspark.sql.Column`, float, list of floats or tuple of floats - percentage in decimal (must be between 0.0 and 1.0). - When percentage is an array, each value of the percentage array must be between 0.0 and 1.0. - In this case, returns the approximate percentile array of column col - at the given percentage array. - accuracy : :class:`~pyspark.sql.Column` or int - is a positive numeric literal which controls approximation accuracy - at the cost of memory. Higher value of accuracy yields better accuracy, - 1.0/accuracy is the relative error of the approximation. (default: 10000). + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + comparator : callable, optional + A binary ``(Column, Column) -> Column: ...``. + The comparator will take two + arguments representing two elements of the array. It returns a negative integer, 0, or a + positive integer as the first element is less than, equal to, or greater than the second + element. If the comparator function returns null, the function will fail and raise an error. Returns ------- :class:`~pyspark.sql.Column` - approximate `percentile` of the numeric column. + sorted array. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.median` - :meth:`pyspark.sql.functions.percentile` - :meth:`pyspark.sql.functions.percentile_approx` + :meth:`pyspark.sql.functions.sort_array` Examples -------- - >>> from pyspark.sql import functions as sf - >>> key = (sf.col("id") % 3).alias("key") - >>> value = (sf.randn(42) + key * 10).alias("value") - >>> df = spark.range(0, 1000, 1, 1).select(key, value) - >>> df.select( - ... sf.approx_percentile("value", [0.25, 0.5, 0.75], 1000000) - ... ).show(truncate=False) - +----------------------------------------------------------+ - |approx_percentile(value, array(0.25, 0.5, 0.75), 1000000) | - +----------------------------------------------------------+ - |[0.7264430125286..., 9.98975299938..., 19.335304783039...]| - +----------------------------------------------------------+ - - >>> df.groupBy("key").agg( - ... sf.approx_percentile("value", sf.lit(0.5), sf.lit(1000000)) - ... ).sort("key").show() - +---+--------------------------------------+ - |key|approx_percentile(value, 0.5, 1000000)| - +---+--------------------------------------+ - | 0| -0.03519435193070...| - | 1| 9.990389751837...| - | 2| 19.967859769284...| - +---+--------------------------------------+ + >>> df = spark.createDataFrame([([2, 1, None, 3],),([1],),([],)], ['data']) + >>> df.select(array_sort(df.data).alias('r')).collect() + [Row(r=[1, 2, 3, None]), Row(r=[1]), Row(r=[])] + >>> df = spark.createDataFrame([(["foo", "foobar", None, "bar"],),(["foo"],),([],)], ['data']) + >>> df.select(array_sort( + ... "data", + ... lambda x, y: when(x.isNull() | y.isNull(), lit(0)).otherwise(length(y) - length(x)) + ... ).alias("r")).collect() + [Row(r=['foobar', 'foo', None, 'bar']), Row(r=['foo']), Row(r=[])] """ - percentage = lit(list(percentage)) if isinstance(percentage, (list, tuple)) else lit(percentage) - return _invoke_function_over_columns("approx_percentile", col, percentage, lit(accuracy)) + if comparator is None: + return _invoke_function_over_columns("array_sort", col) + else: + return _invoke_higher_order_function("array_sort", [col], [comparator]) @_try_remote_functions -def any_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: - """Returns some value of `col` for a group of rows. +def shuffle(col: "ColumnOrName", seed: Optional[Union[Column, int]] = None) -> Column: + """ + Array function: Generates a random permutation of the given array. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column of any type. - ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional - if first value is null then look for first non-null value. - A column that evaluates to a boolean. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + The name of the column or expression to be shuffled. + A column that evaluates to an array. + seed : :class:`~pyspark.sql.Column` or int, optional + Seed value for the random generator. + A column that evaluates to an integer or long. Must be a constant. + + .. versionadded:: 4.0.0 Returns ------- :class:`~pyspark.sql.Column` - some value of `col` for a group of rows. + A new column that contains an array of elements in random order. + Returns a column that evaluates to an array. + + Notes + ----- + The `shuffle` function is non-deterministic, meaning the order of the output array + can be different for each execution. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.select(sf.any_value('c1'), sf.any_value('c2')).show() - +-------------+-------------+ - |any_value(c1)|any_value(c2)| - +-------------+-------------+ - | NULL| 1| - +-------------+-------------+ + Example 1: Shuffling a simple array - >>> df.select(sf.any_value('c1', True), sf.any_value('c2', True)).show() + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT ARRAY(1, 20, 3, 5) AS data") + >>> df.select("*", sf.shuffle(df.data, sf.lit(123))).show() # doctest: +SKIP +-------------+-------------+ - |any_value(c1)|any_value(c2)| + | data|shuffle(data)| +-------------+-------------+ - | a| 1| + |[1, 20, 3, 5]|[5, 1, 20, 3]| +-------------+-------------+ - """ - if ignoreNulls is None: - return _invoke_function_over_columns("any_value", col) - else: - ignoreNulls = _enum_to_value(ignoreNulls) - ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls - return _invoke_function_over_columns("any_value", col, ignoreNulls) - - -@_try_remote_functions -def first_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: - """Returns the first value of `col` for a group of rows. It will return the first non-null - value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. - .. versionadded:: 3.5.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column of any type. - ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional - if first value is null then look for first non-null value. - A column that evaluates to a boolean. Must be a constant. + Example 2: Shuffling an array with null values - Returns - ------- - :class:`~pyspark.sql.Column` - some value of `col` for a group of rows. + >>> import pyspark.sql.functions as sf + >>> df = spark.sql("SELECT ARRAY(1, 20, NULL, 5) AS data") + >>> df.select("*", sf.shuffle(sf.col("data"), 234)).show() # doctest: +SKIP + +----------------+----------------+ + | data| shuffle(data)| + +----------------+----------------+ + |[1, 20, NULL, 5]|[NULL, 5, 20, 1]| + +----------------+----------------+ - See Also - -------- - :meth:`pyspark.sql.functions.last_value` - :meth:`pyspark.sql.functions.nth_value` + Example 3: Shuffling an array with duplicate values - Examples - -------- >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] - ... ).select(sf.first_value('a'), sf.first_value('b')).show() - +--------------+--------------+ - |first_value(a)|first_value(b)| - +--------------+--------------+ - | NULL| 1| - +--------------+--------------+ + >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") + >>> df.select("*", sf.shuffle("data", 345)).show() # doctest: +SKIP + +------------------+------------------+ + | data| shuffle(data)| + +------------------+------------------+ + |[1, 2, 2, 3, 3, 3]|[2, 3, 3, 1, 2, 3]| + +------------------+------------------+ + + Example 4: Shuffling an array with random seed >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [(None, 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["a", "b"] - ... ).select(sf.first_value('a', True), sf.first_value('b', True)).show() - +--------------+--------------+ - |first_value(a)|first_value(b)| - +--------------+--------------+ - | a| 1| - +--------------+--------------+ + >>> df = spark.sql("SELECT ARRAY(1, 2, 2, 3, 3, 3) AS data") + >>> df.select("*", sf.shuffle("data")).show() # doctest: +SKIP + +------------------+------------------+ + | data| shuffle(data)| + +------------------+------------------+ + |[1, 2, 2, 3, 3, 3]|[3, 3, 2, 3, 2, 1]| + +------------------+------------------+ """ - if ignoreNulls is None: - return _invoke_function_over_columns("first_value", col) + if seed is not None: + return _invoke_function_over_columns("shuffle", col, lit(seed)) else: - ignoreNulls = _enum_to_value(ignoreNulls) - ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls - return _invoke_function_over_columns("first_value", col, ignoreNulls) + return _invoke_function_over_columns("shuffle", col) @_try_remote_functions -def last_value(col: "ColumnOrName", ignoreNulls: Optional[Union[bool, Column]] = None) -> Column: - """Returns the last value of `col` for a group of rows. It will return the last non-null - value it sees when `ignoreNulls` is set to true. If all values are null, then null is returned. +def reverse(col: "ColumnOrName") -> Column: + """ + Collection function: returns a reversed string, a binary value with bytes in reverse order, + or an array with elements in reverse order. - .. versionadded:: 3.5.0 + .. versionadded:: 1.5.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. versionchanged:: 4.2.0 + Added support for binary type. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column of any type. - ignoreNulls : :class:`~pyspark.sql.Column` or bool, optional - if first value is null then look for first non-null value. - A column that evaluates to a boolean. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the element to be reversed. + A column that evaluates to a string, binary, or array. Returns ------- :class:`~pyspark.sql.Column` - some value of `col` for a group of rows. - - See Also - -------- - :meth:`pyspark.sql.functions.first_value` - :meth:`pyspark.sql.functions.nth_value` + A new column that contains a reversed string, a binary value with bytes in reverse order, + or an array with elements in reverse order. + Returns a column of the same type as the input. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] - ... ).select(sf.last_value('a'), sf.last_value('b')).show() - +-------------+-------------+ - |last_value(a)|last_value(b)| - +-------------+-------------+ - | NULL| 2| - +-------------+-------------+ + Example 1: Reverse a string >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), (None, 2)], ["a", "b"] - ... ).select(sf.last_value('a', True), sf.last_value('b', True)).show() - +-------------+-------------+ - |last_value(a)|last_value(b)| - +-------------+-------------+ - | b| 2| - +-------------+-------------+ + >>> df = spark.createDataFrame([('Spark SQL',)], ['data']) + >>> df.select(sf.reverse(df.data)).show() + +-------------+ + |reverse(data)| + +-------------+ + | LQS krapS| + +-------------+ + + Example 2: Reverse an array + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([2, 1, 3],) ,([1],) ,([],)], ['data']) + >>> df.select(sf.reverse(df.data)).show() + +-------------+ + |reverse(data)| + +-------------+ + | [3, 1, 2]| + | [1]| + | []| + +-------------+ + + Example 3: Reverse binary data + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytearray(b"\\xCA\\xFE"),)], "data: binary") + >>> df.select(sf.hex(sf.reverse(df.data))).show() + +------------------+ + |hex(reverse(data))| + +------------------+ + | FECA| + +------------------+ """ - if ignoreNulls is None: - return _invoke_function_over_columns("last_value", col) - else: - ignoreNulls = _enum_to_value(ignoreNulls) - ignoreNulls = lit(ignoreNulls) if isinstance(ignoreNulls, bool) else ignoreNulls - return _invoke_function_over_columns("last_value", col, ignoreNulls) + return _invoke_function_over_columns("reverse", col) @_try_remote_functions -def count_if(col: "ColumnOrName") -> Column: +def flatten(col: "ColumnOrName") -> Column: """ - Aggregate function: Returns the number of `TRUE` values for the `col`. + Array function: creates a single array from an array of arrays. + If a structure of nested arrays is deeper than two levels, + only one level of nesting is removed. - .. versionadded:: 3.5.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to a boolean. + col : :class:`~pyspark.sql.Column` or str + The name of the column or expression to be flattened. Returns ------- :class:`~pyspark.sql.Column` - the number of `TRUE` values for the `col`. - - See Also - -------- - :meth:`pyspark.sql.functions.count` + A new column that contains the flattened array. Examples -------- - Example 1: Counting the number of even numbers in a numeric column + Example 1: Flattening a simple nested array >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.select(sf.count_if(sf.col('c2') % 2 == 0)).show() - +------------------------+ - |count_if(((c2 % 2) = 0))| - +------------------------+ - | 3| - +------------------------+ + >>> df = spark.createDataFrame([([[1, 2, 3], [4, 5], [6]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show() + +------------------+ + | flatten(data)| + +------------------+ + |[1, 2, 3, 4, 5, 6]| + +------------------+ - Example 2: Counting the number of rows where a string column starts with a certain letter + Example 2: Flattening an array with null values >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("apple",), ("banana",), ("cherry",), ("apple",), ("banana",)], ["fruit"]) - >>> df.select(sf.count_if(sf.col('fruit').startswith('a'))).show() - +------------------------------+ - |count_if(startswith(fruit, a))| - +------------------------------+ - | 2| - +------------------------------+ + >>> df = spark.createDataFrame([([None, [4, 5]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show() + +-------------+ + |flatten(data)| + +-------------+ + | NULL| + +-------------+ - Example 3: Counting the number of rows where a numeric column is greater than a certain value + Example 3: Flattening an array with more than two levels of nesting >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,), (2,), (3,), (4,), (5,)], ["num"]) - >>> df.select(sf.count_if(sf.col('num') > 3)).show() - +-------------------+ - |count_if((num > 3))| - +-------------------+ - | 2| - +-------------------+ + >>> df = spark.createDataFrame([([[[1, 2], [3, 4]], [[5, 6], [7, 8]]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show(truncate=False) + +--------------------------------+ + |flatten(data) | + +--------------------------------+ + |[[1, 2], [3, 4], [5, 6], [7, 8]]| + +--------------------------------+ - Example 4: Counting the number of rows where a boolean column is True + Example 4: Flattening an array with mixed types >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(True,), (False,), (True,), (False,), (True,)], ["b"]) - >>> df.select(sf.count('b'), sf.count_if('b')).show() - +--------+-----------+ - |count(b)|count_if(b)| - +--------+-----------+ - | 5| 3| - +--------+-----------+ + >>> df = spark.createDataFrame([([['a', 'b', 'c'], [1, 2, 3]],)], ['data']) + >>> df.select(sf.flatten(df.data)).show() + +------------------+ + | flatten(data)| + +------------------+ + |[a, b, c, 1, 2, 3]| + +------------------+ """ - return _invoke_function_over_columns("count_if", col) + return _invoke_function_over_columns("flatten", col) @_try_remote_functions -def histogram_numeric(col: "ColumnOrName", nBins: Column) -> Column: - """Computes a histogram on numeric 'col' using nb bins. - The return value is an array of (x,y) pairs representing the centers of the - histogram's bins. As the value of 'nb' is increased, the histogram approximation - gets finer-grained, but may yield artifacts around outliers. In practice, 20-40 - histogram bins appear to work well, with more bins being required for skewed or - smaller datasets. Note that this function creates a histogram with non-uniform - bin widths. It offers no guarantees in terms of the mean-squared-error of the - histogram, but in practice is comparable to the histograms produced by the R/S-Plus - statistical computing packages. Note: the output type of the 'x' field in the return value is - propagated from the input value consumed in the aggregate function. +def map_contains_key(col: "ColumnOrName", value: Any) -> Column: + """ + Map function: Returns true if the map contains the key. - .. versionadded:: 3.5.0 + .. versionadded:: 3.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - nBins : :class:`~pyspark.sql.Column` - number of Histogram columns. + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the map. + value : + A literal value, or a :class:`~pyspark.sql.Column` expression. + + .. versionchanged:: 4.0.0 + `value` now also accepts a Column type. Returns ------- :class:`~pyspark.sql.Column` - a histogram on numeric 'col' using nb bins. + True if key is in the map and False otherwise. Examples -------- + Example 1: The key is in the map + >>> from pyspark.sql import functions as sf - >>> df = spark.range(100, numPartitions=1) - >>> df.select(sf.histogram_numeric('id', sf.lit(5))).show(truncate=False) - +-----------------------------------------------------------+ - |histogram_numeric(id, 5) | - +-----------------------------------------------------------+ - |[{11, 25.0}, {36, 24.0}, {59, 23.0}, {84, 25.0}, {98, 3.0}]| - +-----------------------------------------------------------+ - """ - return _invoke_function_over_columns("histogram_numeric", col, nBins) + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.map_contains_key("data", 1)).show() + +-------------------------+ + |map_contains_key(data, 1)| + +-------------------------+ + | true| + +-------------------------+ + Example 2: The key is not in the map -@_try_remote_functions -def hll_sketch_agg( - col: "ColumnOrName", - lgConfigK: Optional[Union[int, Column]] = None, -) -> Column: + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.map_contains_key("data", -1)).show() + +--------------------------+ + |map_contains_key(data, -1)| + +--------------------------+ + | false| + +--------------------------+ + + Example 3: Check for key using a column + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data, 1 as key") + >>> df.select(sf.map_contains_key("data", sf.col("key"))).show() + +---------------------------+ + |map_contains_key(data, key)| + +---------------------------+ + | true| + +---------------------------+ """ - Aggregate function: returns the updatable binary representation of the Datasketches - HllSketch configured with lgConfigK arg. + return _invoke_function_over_columns("map_contains_key", col, lit(value)) - .. versionadded:: 3.5.0 + +@_try_remote_functions +def map_keys(col: "ColumnOrName") -> Column: + """ + Map function: Returns an unordered array containing the keys of the map. + + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to an integer, long, string, or binary. - lgConfigK : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of K, where K is the number of buckets or slots for the HllSketch. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or str + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the HllSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.hll_union` - :meth:`pyspark.sql.functions.hll_union_agg` - :meth:`pyspark.sql.functions.hll_sketch_estimate` + Keys of the map as an array. Examples -------- + Example 1: Extracting keys from a simple map + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() - +----------------------------------------------+ - |hll_sketch_estimate(hll_sketch_agg(value, 12))| - +----------------------------------------------+ - | 3| - +----------------------------------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_keys("data"))).show() + +--------------------------------+ + |sort_array(map_keys(data), true)| + +--------------------------------+ + | [1, 2]| + +--------------------------------+ - >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value", 12))).show() - +----------------------------------------------+ - |hll_sketch_estimate(hll_sketch_agg(value, 12))| - +----------------------------------------------+ - | 3| - +----------------------------------------------+ + Example 2: Extracting keys from a map with complex keys + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(array(1, 2), 'a', array(3, 4), 'b') as data") + >>> df.select(sf.sort_array(sf.map_keys("data"))).show() + +--------------------------------+ + |sort_array(map_keys(data), true)| + +--------------------------------+ + | [[1, 2], [3, 4]]| + +--------------------------------+ + + Example 3: Extracting keys from a map with duplicate keys + + >>> from pyspark.sql import functions as sf + >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") + >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") + >>> df.select(sf.map_keys("data")).show() + +--------------+ + |map_keys(data)| + +--------------+ + | [1]| + +--------------+ + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) + + Example 4: Extracting keys from an empty map + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map() as data") + >>> df.select(sf.map_keys("data")).show() + +--------------+ + |map_keys(data)| + +--------------+ + | []| + +--------------+ """ - if lgConfigK is None: - return _invoke_function_over_columns("hll_sketch_agg", col) - else: - return _invoke_function_over_columns("hll_sketch_agg", col, lit(lgConfigK)) + return _invoke_function_over_columns("map_keys", col) @_try_remote_functions -def hll_union_agg( - col: "ColumnOrName", - allowDifferentLgConfigK: Optional[Union[bool, Column]] = None, -) -> Column: +def map_values(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the updatable binary representation of the Datasketches - HllSketch, generated by merging previously created Datasketches HllSketch instances - via a Datasketches Union instance. Throws an exception if sketches have different - lgConfigK values and allowDifferentLgConfigK is unset or set to false. + Map function: Returns an unordered array containing the values of the map. - .. versionadded:: 3.5.0 + .. versionadded:: 2.3.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - allowDifferentLgConfigK : :class:`~pyspark.sql.Column` or bool, optional - Allow sketches with different lgConfigK values to be merged (defaults to false). + col : :class:`~pyspark.sql.Column` or str + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged HllSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.hll_union` - :meth:`pyspark.sql.functions.hll_sketch_agg` - :meth:`pyspark.sql.functions.hll_sketch_estimate` + Values of the map as an array. Examples -------- + Example 1: Extracting values from a simple map + >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,2,3], "INT") - >>> df1 = df1.agg(sf.hll_sketch_agg("value").alias("sketch")) - >>> df2 = spark.createDataFrame([4,5,5,6], "INT") - >>> df2 = df2.agg(sf.hll_sketch_agg("value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch"))).show() - +-------------------------------------------------+ - |hll_sketch_estimate(hll_union_agg(sketch, false))| - +-------------------------------------------------+ - | 6| - +-------------------------------------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_values("data"))).show() + +----------------------------------+ + |sort_array(map_values(data), true)| + +----------------------------------+ + | [a, b]| + +----------------------------------+ - >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch", False))).show() - +-------------------------------------------------+ - |hll_sketch_estimate(hll_union_agg(sketch, false))| - +-------------------------------------------------+ - | 6| - +-------------------------------------------------+ - """ - if allowDifferentLgConfigK is None: - return _invoke_function_over_columns("hll_union_agg", col) - else: - return _invoke_function_over_columns("hll_union_agg", col, lit(allowDifferentLgConfigK)) + Example 2: Extracting values from a map with complex values + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, array('a', 'b'), 2, array('c', 'd')) as data") + >>> df.select(sf.sort_array(sf.map_values("data"))).show() + +----------------------------------+ + |sort_array(map_values(data), true)| + +----------------------------------+ + | [[a, b], [c, d]]| + +----------------------------------+ -@_try_remote_functions -def theta_sketch_agg( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - ThetaSketch with the values in the input column configured with lgNomEntries nominal entries. + Example 3: Extracting values from a map with null values - .. versionadded:: 4.1.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, null, 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_values("data"))).show() + +----------------------------------+ + |sort_array(map_values(data), true)| + +----------------------------------+ + | [NULL, b]| + +----------------------------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - A column that evaluates to an array, binary, double, float, integer, long, or string. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries, where nominal entries is the size of the sketch - (must be between 4 and 26, defaults to 12). - A column that evaluates to an integer. + Example 4: Extracting values from a map with duplicate values - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the ThetaSketch. + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'a') as data") + >>> df.select(sf.map_values("data")).show() + +----------------+ + |map_values(data)| + +----------------+ + | [a, a]| + +----------------+ - See Also - -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_intersection` - :meth:`pyspark.sql.functions.theta_difference` - :meth:`pyspark.sql.functions.theta_union_agg` - :meth:`pyspark.sql.functions.theta_intersection_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + Example 5: Extracting values from an empty map - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_sketch_agg(value, 12))| - +--------------------------------------------------+ - | 3| - +--------------------------------------------------+ - - >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value", 15))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_sketch_agg(value, 15))| - +--------------------------------------------------+ - | 3| - +--------------------------------------------------+ + >>> df = spark.sql("SELECT map() as data") + >>> df.select(sf.map_values("data")).show() + +----------------+ + |map_values(data)| + +----------------+ + | []| + +----------------+ """ - fn = "theta_sketch_agg" - if lgNomEntries is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) + return _invoke_function_over_columns("map_values", col) @_try_remote_functions -def theta_union_agg( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, -) -> Column: +def map_entries(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - ThetaSketch that is the union of the Theta sketches in the input column. + Map function: Returns an unordered array of all entries in the given map. - .. versionadded:: 4.1.0 + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries for the union operation - (must be between 4 and 26, defaults to 12) + col : :class:`~pyspark.sql.Column` or str + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged ThetaSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + An array of key value pairs as a struct type Examples -------- + Example 1: Extracting entries from a simple map + >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,2,3], "INT") - >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df2 = spark.createDataFrame([4,5,5,6], "INT") - >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.theta_sketch_estimate(sf.theta_union_agg("sketch"))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_union_agg(sketch, 12))| - +--------------------------------------------------+ - | 6| - +--------------------------------------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as data") + >>> df.select(sf.sort_array(sf.map_entries("data"))).show() + +-----------------------------------+ + |sort_array(map_entries(data), true)| + +-----------------------------------+ + | [{1, a}, {2, b}]| + +-----------------------------------+ + + Example 2: Extracting entries from a map with complex keys and values + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(array(1, 2), array('a', 'b'), " + ... "array(3, 4), array('c', 'd')) as data") + >>> df.select(sf.sort_array(sf.map_entries("data"))).show(truncate=False) + +------------------------------------+ + |sort_array(map_entries(data), true) | + +------------------------------------+ + |[{[1, 2], [a, b]}, {[3, 4], [c, d]}]| + +------------------------------------+ + + Example 3: Extracting entries from a map with duplicate keys + + >>> from pyspark.sql import functions as sf + >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") + >>> df = spark.sql("SELECT map(1, 'a', 1, 'b') as data") + >>> df.select(sf.map_entries("data")).show() + +-----------------+ + |map_entries(data)| + +-----------------+ + | [{1, b}]| + +-----------------+ + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) + + Example 4: Extracting entries from an empty map + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map() as data") + >>> df.select(sf.map_entries("data")).show() + +-----------------+ + |map_entries(data)| + +-----------------+ + | []| + +-----------------+ """ - fn = "theta_union_agg" - if lgNomEntries is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) + return _invoke_function_over_columns("map_entries", col) @_try_remote_functions -def theta_intersection_agg(col: "ColumnOrName") -> Column: +def map_from_entries(col: "ColumnOrName") -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - ThetaSketch that is the intersection of the Theta sketches in the input column + Map function: Transforms an array of key-value pair entries (structs with two fields) + into a map. The first field of each entry is used as the key and the second field + as the value in the resulting map column - .. versionadded:: 4.1.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + col : :class:`~pyspark.sql.Column` or str + Name of column or expression Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected ThetaSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.theta_intersection` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + A map created from the given array of entries. Examples -------- + Example 1: Basic usage of map_from_entries + >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,2,3], "INT") - >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df2 = spark.createDataFrame([2,3,3,4], "INT") - >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.theta_sketch_estimate(sf.theta_intersection_agg("sketch"))).show() - +-----------------------------------------------------+ - |theta_sketch_estimate(theta_intersection_agg(sketch))| - +-----------------------------------------------------+ - | 2| - +-----------------------------------------------------+ + >>> df = spark.sql("SELECT array(struct(1, 'a'), struct(2, 'b')) as data") + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {1 -> a, 2 -> b}| + +----------------------+ + + Example 2: map_from_entries with null values + + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT array(struct(1, null), struct(2, 'b')) as data") + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {1 -> NULL, 2 -> b}| + +----------------------+ + + Example 3: map_from_entries with a DataFrame + + >>> from pyspark.sql import Row, functions as sf + >>> df = spark.createDataFrame([([Row(1, "a"), Row(2, "b")],), ([Row(3, "c")],)], ['data']) + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {1 -> a, 2 -> b}| + | {3 -> c}| + +----------------------+ + + Example 4: map_from_entries with empty array + + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, StringType, IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", ArrayType( + ... StructType([ + ... StructField("key", IntegerType()), + ... StructField("value", StringType()) + ... ]) + ... ), True) + ... ]) + >>> df = spark.createDataFrame([([],)], schema=schema) + >>> df.select(sf.map_from_entries(df.data)).show() + +----------------------+ + |map_from_entries(data)| + +----------------------+ + | {}| + +----------------------+ """ - fn = "theta_intersection_agg" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_columns("map_from_entries", col) @_try_remote_functions -def tuple_sketch_agg_double( - key: "ColumnOrName", - summary: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def array_repeat(col: "ColumnOrName", count: Union["ColumnOrName", int]) -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch with double summaries built from the key and summary columns. + Array function: creates an array containing a column repeated count times. - .. versionadded:: 4.2.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - key : :class:`~pyspark.sql.Column` or column name - The column containing key values. - A column that evaluates to an array, binary, double, float, integer, long, or string. - summary : :class:`~pyspark.sql.Column` or column name - The column containing double summary values. - A column that evaluates to a double. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + col : :class:`~pyspark.sql.Column` or str + The name of the column or an expression that represents the element to be repeated. + A column of any type. + count : :class:`~pyspark.sql.Column` or str or int + The name of the column, an expression, + or an integer that represents the number of times to repeat the element. A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the TupleSketch. + A new column that contains an array of repeated elements. + Returns a column that evaluates to an array. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` - :meth:`pyspark.sql.functions.tuple_sketch_summary_double` - :meth:`pyspark.sql.functions.tuple_union_agg_double` + :meth:`pyspark.sql.functions.array` Examples -------- + Example 1: Usage with string + >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +--------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| - +--------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------+ - """ - fn = "tuple_sketch_agg_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) + >>> df = spark.createDataFrame([('ab',)], ['data']) + >>> df.select(sf.array_repeat(df.data, 3)).show() + +---------------------+ + |array_repeat(data, 3)| + +---------------------+ + | [ab, ab, ab]| + +---------------------+ - return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) + Example 2: Usage with integer + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(3,)], ['data']) + >>> df.select(sf.array_repeat(df.data, 2)).show() + +---------------------+ + |array_repeat(data, 2)| + +---------------------+ + | [3, 3]| + +---------------------+ -@_try_remote_functions -def tuple_sketch_agg_integer( - key: "ColumnOrName", - summary: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch with integer summaries built from the key and summary columns. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - key : :class:`~pyspark.sql.Column` or column name - The column containing key values. - A column that evaluates to an array, binary, double, float, integer, long, or string. - summary : :class:`~pyspark.sql.Column` or column name - The column containing integer summary values. - A column that evaluates to an integer. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + Example 3: Usage with array - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the TupleSketch. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(['apple', 'banana'],)], ['data']) + >>> df.select(sf.array_repeat(df.data, 2)).show(truncate=False) + +----------------------------------+ + |array_repeat(data, 2) | + +----------------------------------+ + |[[apple, banana], [apple, banana]]| + +----------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` - :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` - :meth:`pyspark.sql.functions.tuple_union_agg_integer` + Example 4: Usage with null - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +----------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| - +----------------------------------------------------------------------------+ - | 2.0| - +----------------------------------------------------------------------------+ + >>> from pyspark.sql.types import IntegerType, StructType, StructField + >>> schema = StructType([ + ... StructField("data", IntegerType(), True) + ... ]) + >>> df = spark.createDataFrame([(None, )], schema=schema) + >>> df.select(sf.array_repeat(df.data, 3)).show() + +---------------------+ + |array_repeat(data, 3)| + +---------------------+ + | [NULL, NULL, NULL]| + +---------------------+ """ - fn = "tuple_sketch_agg_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) + count = _enum_to_value(count) + count = lit(count) if isinstance(count, int) else count - return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) + return _invoke_function_over_columns("array_repeat", col, count) @_try_remote_functions -def tuple_union_agg_double( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def arrays_zip(*cols: "ColumnOrName") -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the union of the double TupleSketch objects in the input column. + Array function: Returns a merged array of structs in which the N-th struct contains all + N-th values of input arrays. If one of the arrays is shorter than others then + the resulting struct type value will be a `null` for missing elements. - .. versionadded:: 4.2.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + cols : :class:`~pyspark.sql.Column` or str + Columns of arrays to be merged. + A column that evaluates to an array. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_double` + Merged array of entries. + Returns a column that evaluates to an array. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(3, 30.0), (4, 40.0)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_union_agg_double("sketch"))).show() - +---------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_union_agg_double(sketch, 12, sum))| - +---------------------------------------------------------------------+ - | 4.0| - +---------------------------------------------------------------------+ - """ - fn = "tuple_union_agg_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) + Example 1: Zipping two arrays of the same length - return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2, 3], ['a', 'b', 'c'])], ['nums', 'letters']) + >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) + +-------------------------+ + |arrays_zip(nums, letters)| + +-------------------------+ + |[{1, a}, {2, b}, {3, c}] | + +-------------------------+ -@_try_remote_functions -def tuple_union_agg_integer( - col: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the union of the integer TupleSketch objects in the input column. + Example 2: Zipping arrays of different lengths - .. versionadded:: 4.2.0 + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([([1, 2], ['a', 'b', 'c'])], ['nums', 'letters']) + >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) + +---------------------------+ + |arrays_zip(nums, letters) | + +---------------------------+ + |[{1, a}, {2, b}, {NULL, c}]| + +---------------------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + Example 3: Zipping more than two arrays - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame( + ... [([1, 2], ['a', 'b'], [True, False])], ['nums', 'letters', 'bools']) + >>> df.select(sf.arrays_zip(df.nums, df.letters, df.bools)).show(truncate=False) + +--------------------------------+ + |arrays_zip(nums, letters, bools)| + +--------------------------------+ + |[{1, a, true}, {2, b, false}] | + +--------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_integer` + Example 4: Zipping arrays with null values - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(3, 30), (4, 40)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_union_agg_integer("sketch"))).show() - +-----------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch, 12, sum))| - +-----------------------------------------------------------------------+ - | 4.0| - +-----------------------------------------------------------------------+ + >>> df = spark.createDataFrame([([1, 2, None], ['a', None, 'c'])], ['nums', 'letters']) + >>> df.select(sf.arrays_zip(df.nums, df.letters)).show(truncate=False) + +------------------------------+ + |arrays_zip(nums, letters) | + +------------------------------+ + |[{1, a}, {2, NULL}, {NULL, c}]| + +------------------------------+ """ - fn = "tuple_union_agg_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) + return _invoke_function_over_seq_of_columns("arrays_zip", cols) - return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) + +@overload +def map_concat(*cols: "ColumnOrName") -> Column: ... + + +@overload +def map_concat(__cols: Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]) -> Column: ... @_try_remote_functions -def tuple_intersection_agg_double( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, +def map_concat( + *cols: Union["ColumnOrName", Union[Sequence["ColumnOrName"], Tuple["ColumnOrName", ...]]], ) -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the intersection of the double TupleSketch objects in the input column. + Map function: Returns the union of all given maps. - .. versionadded:: 4.2.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + cols : :class:`~pyspark.sql.Column` or str + Column names or :class:`~pyspark.sql.Column` Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + A map of merged entries from other maps. - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_intersection_double` + Notes + ----- + For duplicate keys in input maps, the handling is governed by `spark.sql.mapKeyDedupPolicy`. + By default, it throws an exception. If set to `LAST_WIN`, it uses the last map's value. Examples -------- + Example 1: Basic usage of map_concat + >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0), (3, 30.0)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(2, 40.0), (3, 50.0), (4, 60.0)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_intersection_agg_double("sketch"))).show() - +------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch, sum))| - +------------------------------------------------------------------------+ - | 2.0| - +------------------------------------------------------------------------+ - """ - fn = "tuple_intersection_agg_double" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, 'c') as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +------------------------+ + |map_concat(map1, map2) | + +------------------------+ + |{1 -> a, 2 -> b, 3 -> c}| + +------------------------+ + Example 2: map_concat with overlapping keys -@_try_remote_functions -def tuple_intersection_agg_integer( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - TupleSketch that is the intersection of the integer TupleSketch objects in the input column. + >>> from pyspark.sql import functions as sf + >>> originalmapKeyDedupPolicy = spark.conf.get("spark.sql.mapKeyDedupPolicy") + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", "LAST_WIN") + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(2, 'c', 3, 'd') as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +------------------------+ + |map_concat(map1, map2) | + +------------------------+ + |{1 -> a, 2 -> c, 3 -> d}| + +------------------------+ + >>> spark.conf.set("spark.sql.mapKeyDedupPolicy", originalmapKeyDedupPolicy) - .. versionadded:: 4.2.0 + Example 3: map_concat with three maps - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary TupleSketch representations. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a') as map1, map(2, 'b') as map2, map(3, 'c') as map3") + >>> df.select(sf.map_concat("map1", "map2", "map3")).show(truncate=False) + +----------------------------+ + |map_concat(map1, map2, map3)| + +----------------------------+ + |{1 -> a, 2 -> b, 3 -> c} | + +----------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + Example 4: map_concat with empty map - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_intersection_integer` + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map() as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +----------------------+ + |map_concat(map1, map2)| + +----------------------+ + |{1 -> a, 2 -> b} | + +----------------------+ + + Example 5: map_concat with null values - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([(1, 10), (2, 20), (3, 30)], ["key", "value"]) - >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df2 = spark.createDataFrame([(2, 40), (3, 50), (4, 60)], ["key", "value"]) - >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) - >>> df3 = df1.union(df2) - >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_agg_integer("sketch"))).show() - +--------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch, sum))| - +--------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------+ + >>> df = spark.sql("SELECT map(1, 'a', 2, 'b') as map1, map(3, null) as map2") + >>> df.select(sf.map_concat("map1", "map2")).show(truncate=False) + +---------------------------+ + |map_concat(map1, map2) | + +---------------------------+ + |{1 -> a, 2 -> b, 3 -> NULL}| + +---------------------------+ """ - fn = "tuple_intersection_agg_integer" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + if len(cols) == 1 and isinstance(cols[0], (list, set)): + cols = cols[0] # type: ignore[assignment] + return _invoke_function_over_seq_of_columns("map_concat", cols) # type: ignore[arg-type] @_try_remote_functions -def kll_sketch_agg_bigint( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, +def sequence( + start: "ColumnOrName", stop: "ColumnOrName", step: Optional["ColumnOrName"] = None ) -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - KllLongsSketch built with the values in the input column. The optional k parameter - controls the size and accuracy of the sketch (default 200, range 8-65535). + Array function: Generate a sequence of integers from `start` to `stop`, incrementing by `step`. + If `step` is not set, the function increments by 1 if `start` is less than or equal to `stop`, + otherwise it decrements by 1. - .. versionadded:: 4.1.0 + .. versionadded:: 2.4.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing bigint values to aggregate. - A column that evaluates to an integral. - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (default 200, range 8-65535) - A column that evaluates to an integer. Must be a constant. + start : :class:`~pyspark.sql.Column` or str + The starting value (inclusive) of the sequence. + A column that evaluates to an integral, date, or timestamp. + stop : :class:`~pyspark.sql.Column` or str + The last value (inclusive) of the sequence. + A column that evaluates to an integral, date, or timestamp. + step : :class:`~pyspark.sql.Column` or str, optional + The value to add to the current element to get the next element in the sequence. + The default is 1 if `start` is less than or equal to `stop`, otherwise -1. + A column that evaluates to an integral or interval. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the KllLongsSketch. + A new column that contains an array of sequence values. + Returns a column that evaluates to an array. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> result = df.agg(sf.kll_sketch_agg_bigint("value")).first()[0] - >>> result is not None and len(result) > 0 - True - """ - fn = "kll_sketch_agg_bigint" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + Example 1: Generating a sequence with default step + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(-2, 2)], ['start', 'stop']) + >>> df.select(sf.sequence(df.start, df.stop)).show() + +---------------------+ + |sequence(start, stop)| + +---------------------+ + | [-2, -1, 0, 1, 2]| + +---------------------+ -@_try_remote_functions -def kll_sketch_agg_float( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, -) -> Column: - """ - Aggregate function: returns the compact binary representation of the Datasketches - KllFloatsSketch built with the values in the input column. The optional k parameter - controls the size and accuracy of the sketch (default 200, range 8-65535). + Example 2: Generating a sequence with a custom step - .. versionadded:: 4.1.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(4, -4, -2)], ['start', 'stop', 'step']) + >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() + +---------------------------+ + |sequence(start, stop, step)| + +---------------------------+ + | [4, 2, 0, -2, -4]| + +---------------------------+ - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing float values to aggregate - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (default 200, range 8-65535) - A column that evaluates to an integer. Must be a constant. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the KllFloatsSketch. + Example 3: Generating a sequence with a negative step - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> result = df.agg(sf.kll_sketch_agg_float("value")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(5, 1, -1)], ['start', 'stop', 'step']) + >>> df.select(sf.sequence(df.start, df.stop, df.step)).show() + +---------------------------+ + |sequence(start, stop, step)| + +---------------------------+ + | [5, 4, 3, 2, 1]| + +---------------------------+ """ - fn = "kll_sketch_agg_float" - if k is None: - return _invoke_function_over_columns(fn, col) + if step is None: + return _invoke_function_over_columns("sequence", start, stop) else: - return _invoke_function_over_columns(fn, col, lit(k)) + return _invoke_function_over_columns("sequence", start, stop, step) @_try_remote_functions -def kll_sketch_agg_double( +def from_csv( col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, + schema: Union[Column, str], + options: Optional[Mapping[str, str]] = None, ) -> Column: """ - Aggregate function: returns the compact binary representation of the Datasketches - KllDoublesSketch built with the values in the input column. The optional k parameter - controls the size and accuracy of the sketch (default 200, range 8-65535). + CSV Function: Parses a column containing a CSV string into a row with the specified schema. + Returns `null` if the string cannot be parsed. - .. versionadded:: 4.1.0 + .. versionadded:: 3.0.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing double values to aggregate. - A column that evaluates to a float or double. - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (default 200, range 8-65535) - A column that evaluates to an integer. Must be a constant. + col : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string. + A column or column name in CSV format. + schema : :class:`~pyspark.sql.Column` or str + A column that evaluates to a string, or a DDL-formatted type string, or a DataType. + A column, or Python string literal with schema in DDL format, to use when parsing the CSV column. + options : dict, optional + Options to control parsing. Accepts the same options as the CSV datasource. + See `Data Source Option `_ + A dict of options. Each key and value is a string. + for the version you use. + + .. # noqa Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the KllDoublesSketch. + A column of parsed CSV values. + Returns a column that evaluates to a struct. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> result = df.agg(sf.kll_sketch_agg_double("value")).first()[0] - >>> result is not None and len(result) > 0 - True - """ - fn = "kll_sketch_agg_double" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) - + Example 1: Parsing a simple CSV string -@_try_remote_functions -def kll_merge_agg_bigint( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, -) -> Column: - """ - Aggregate function: merges binary KllLongsSketch representations and returns the - merged sketch. The optional k parameter controls the size and accuracy of the merged - sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value - from the first input sketch. - - .. versionadded:: 4.1.2 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary KllLongsSketch representations - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (range 8-65535) - A column that evaluates to an integer. Must be a constant. + >>> from pyspark.sql import functions as sf + >>> data = [("1,2,3",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, 3}| + +---------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - The merged binary representation of the KllLongsSketch. + Example 2: Using schema_of_csv to infer the schema - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1,2,3], "INT") - >>> df2 = spark.createDataFrame([4,5,6], "INT") - >>> sketch1 = df1.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch2 = df2.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_bigint("sketch").alias("merged")) - >>> n = merged.select(sf.kll_sketch_get_n_bigint("merged")).first()[0] - >>> n - 6 - """ - fn = "kll_merge_agg_bigint" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + >>> data = [("1,2,3",)] + >>> value = data[0][0] + >>> df.select(sf.from_csv(df.value, sf.schema_of_csv(value))).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, 3}| + +---------------+ + Example 3: Ignoring leading white space in the CSV string -@_try_remote_functions -def kll_merge_agg_float( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, -) -> Column: - """ - Aggregate function: merges binary KllFloatsSketch representations and returns the - merged sketch. The optional k parameter controls the size and accuracy of the merged - sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value - from the first input sketch. + >>> from pyspark.sql import functions as sf + >>> data = [(" abc",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> options = {'ignoreLeadingWhiteSpace': True} + >>> df.select(sf.from_csv(df.value, "s string", options)).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {abc}| + +---------------+ - .. versionadded:: 4.1.2 + Example 4: Parsing a CSV string with a missing value - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary KllFloatsSketch representations - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (range 8-65535) - A column that evaluates to an integer. Must be a constant. + >>> from pyspark.sql import functions as sf + >>> data = [("1,2,",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, NULL}| + +---------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - The merged binary representation of the KllFloatsSketch. + Example 5: Parsing a CSV string with a different delimiter - Examples - -------- >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "FLOAT") - >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "FLOAT") - >>> sketch1 = df1.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch2 = df2.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_float("sketch").alias("merged")) - >>> n = merged.select(sf.kll_sketch_get_n_float("merged")).first()[0] - >>> n - 6 + >>> data = [("1;2;3",)] + >>> df = spark.createDataFrame(data, ("value",)) + >>> options = {'delimiter': ';'} + >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT", options)).show() + +---------------+ + |from_csv(value)| + +---------------+ + | {1, 2, 3}| + +---------------+ """ - fn = "kll_merge_agg_float" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + from pyspark.sql.classic.column import _to_java_column + + if not isinstance(schema, (str, Column)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "schema", + "arg_type": type(schema).__name__, + }, + ) + return _invoke_function( + "from_csv", _to_java_column(col), _to_java_column(lit(schema)), _options_to_str(options) + ) -@_try_remote_functions -def kll_merge_agg_double( - col: "ColumnOrName", - k: Optional[Union[int, Column]] = None, -) -> Column: + +def _unresolved_named_lambda_variable(name: str) -> Column: """ - Aggregate function: merges binary KllDoublesSketch representations and returns the - merged sketch. The optional k parameter controls the size and accuracy of the merged - sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value - from the first input sketch. + Create `o.a.s.sql.expressions.UnresolvedNamedLambdaVariable`, + convert it to o.s.sql.Column and wrap in Python `Column` - .. versionadded:: 4.1.2 + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing binary KllDoublesSketch representations - k : :class:`~pyspark.sql.Column` or int, optional - The k parameter that controls size and accuracy (range 8-65535) - A column that evaluates to an integer. Must be a constant. - - Returns - ------- - :class:`~pyspark.sql.Column` - The merged binary representation of the KllDoublesSketch. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "DOUBLE") - >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "DOUBLE") - >>> sketch1 = df1.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch2 = df2.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_double("sketch").alias("merged")) - >>> n = merged.select(sf.kll_sketch_get_n_double("merged")).first()[0] - >>> n - 6 + name_parts : str """ - fn = "kll_merge_agg_double" - if k is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(k)) + from py4j.java_gateway import JVMView + sc = _get_active_spark_context() + return Column(cast(JVMView, sc._jvm).PythonSQLUtils.unresolvedNamedLambdaVariable(name)) -@_try_remote_functions -def bitmap_construct_agg(col: "ColumnOrName") -> Column: - """ - Returns a bitmap with the positions of the bits set from all the values from the input column. - The input column will most likely be bitmap_bit_position(). - .. versionadded:: 3.5.0 +def _get_lambda_parameters(f: Callable) -> ValuesView[inspect.Parameter]: + signature = inspect.signature(f) + parameters = signature.parameters.values() - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column will most likely be bitmap_bit_position(). - A column that evaluates to a long. + # We should exclude functions that use + # variable args and keyword argnames + # as well as keyword only args + supported_parameter_types = { + inspect.Parameter.POSITIONAL_OR_KEYWORD, + inspect.Parameter.POSITIONAL_ONLY, + } - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` - :meth:`pyspark.sql.functions.bitmap_and_agg` + # Validate that + # function arity is between 1 and 3 + if not (1 <= len(parameters) <= 3): + raise PySparkValueError( + errorClass="WRONG_NUM_ARGS_FOR_HIGHER_ORDER_FUNCTION", + messageParameters={"func_name": f.__name__, "num_args": str(len(parameters))}, + ) - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,),(2,),(3,)], ["a"]) - >>> df.select( - ... sf.bitmap_construct_agg(sf.bitmap_bit_position('a')) - ... ).show() - +--------------------------------------------+ - |bitmap_construct_agg(bitmap_bit_position(a))| - +--------------------------------------------+ - | [07 00 00 00 00 0...| - +--------------------------------------------+ - """ - return _invoke_function_over_columns("bitmap_construct_agg", col) + # and all arguments can be used as positional + if not all(p.kind in supported_parameter_types for p in parameters): + raise PySparkValueError( + errorClass="UNSUPPORTED_PARAM_TYPE_FOR_HIGHER_ORDER_FUNCTION", + messageParameters={"func_name": f.__name__}, + ) + return parameters -@_try_remote_functions -def bitmap_or_agg(col: "ColumnOrName") -> Column: + +def _create_lambda(f: Callable) -> Callable: """ - Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. - The input column should be bitmaps created from bitmap_construct_agg(). + Create `o.a.s.sql.expressions.LambdaFunction` corresponding + to transformation described by f - .. versionadded:: 3.5.0 + :param f: A Python of one of the following forms: + - (Column) -> Column: ... + - (Column, Column) -> Column: ... + - (Column, Column, Column) -> Column: ... + """ + from py4j.java_gateway import JVMView - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_and_agg` + from pyspark.sql.classic.column import _to_seq - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). + parameters = _get_lambda_parameters(f) - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("10",),("20",),("40",)], ["a"]) - >>> df.select(sf.bitmap_or_agg(sf.to_binary(df.a, sf.lit("hex")))).show() - +--------------------------------+ - |bitmap_or_agg(to_binary(a, hex))| - +--------------------------------+ - | [70 00 00 00 00 0...| - +--------------------------------+ - """ - return _invoke_function_over_columns("bitmap_or_agg", col) + sc = _get_active_spark_context() + argnames = ["x", "y", "z"] + args = [_unresolved_named_lambda_variable(arg) for arg in argnames[: len(parameters)]] -@_try_remote_functions -def bitmap_and_agg(col: "ColumnOrName") -> Column: - """ - Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. - The input column should be bitmaps created from bitmap_construct_agg(). + result = f(*args) - .. versionadded:: 4.1.0 + if not isinstance(result, Column): + raise PySparkValueError( + errorClass="HIGHER_ORDER_FUNCTION_SHOULD_RETURN_COLUMN", + messageParameters={"func_name": f.__name__, "return_type": type(result).__name__}, + ) - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` + jexpr = result._jc + jargs = _to_seq(sc, [arg._jc for arg in args]) + return cast(JVMView, sc._jvm).PythonSQLUtils.lambdaFunction(jexpr, jargs) - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0",),("70",),("30",)], ["a"]) - >>> df.select(sf.bitmap_and_agg(sf.to_binary(df.a, sf.lit("hex")))).show() - +---------------------------------+ - |bitmap_and_agg(to_binary(a, hex))| - +---------------------------------+ - | [30 00 00 00 00 0...| - +---------------------------------+ +def _invoke_higher_order_function( + name: str, + cols: Sequence["ColumnOrName"], + funs: Sequence[Callable], +) -> Column: """ - return _invoke_function_over_columns("bitmap_and_agg", col) + Invokes expression identified by name, + (relative to ```org.apache.spark.sql.catalyst.expressions``) + and wraps the result with Column (first Scala one, then Python). + :param name: Name of the expression + :param cols: a list of columns + :param funs: a list of (*Column) -> Column functions. -@_try_remote_functions -def bitmap_xor_agg(col: "ColumnOrName") -> Column: + :return: a Column """ - Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. - The input column should be bitmaps created from bitmap_construct_agg(). + from py4j.java_gateway import JVMView - .. versionadded:: 4.4.0 + from pyspark.sql.classic.column import _to_java_column, _to_seq - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` - :meth:`pyspark.sql.functions.bitmap_and_agg` + sc = _get_active_spark_context() + jfuns = [_create_lambda(f) for f in funs] + jcols = [_to_java_column(c) for c in cols] + return Column(cast(JVMView, sc._jvm).PythonSQLUtils.fn(name, _to_seq(sc, jcols + jfuns))) - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column should be bitmaps created from bitmap_construct_agg(). - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("10",), ("30",), ("40",)], ["a"]) - >>> df.select(sf.bitmap_xor_agg(sf.to_binary(df.a, sf.lit("hex")))).show() - +---------------------------------+ - |bitmap_xor_agg(to_binary(a, hex))| - +---------------------------------+ - | [60 00 00 00 00 0...| - +---------------------------------+ - """ - return _invoke_function_over_columns("bitmap_xor_agg", col) +@overload +def transform(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... -# ---------------------- Window Functions ---------------------- +@overload +def transform(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... @_try_remote_functions -def row_number() -> Column: +def transform( + col: "ColumnOrName", + f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], +) -> Column: """ - Window function: returns a sequential number starting at 1 within a window partition. + Returns an array of elements after applying a transformation to each element in the input array. - .. versionadded:: 1.6.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + a function that is applied to each element of the input array. + Can take one of the following forms: + + - Unary ``(x: Column) -> Column: ...`` + - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is + a 0-based index of the element. + + and can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). + Returns ------- :class:`~pyspark.sql.Column` - the column for calculating row numbers. - - See Also - -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` + a new array of transformed elements. + Returns a column that evaluates to an array. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.range(3) - >>> w = Window.orderBy(df.id.desc()) - >>> df.withColumn("desc_order", sf.row_number().over(w)).show() - +---+----------+ - | id|desc_order| - +---+----------+ - | 2| 1| - | 1| 2| - | 0| 3| - +---+----------+ + >>> df = spark.createDataFrame([(1, [1, 2, 3, 4])], ("key", "values")) + >>> df.select(transform("values", lambda x: x * 2).alias("doubled")).show() + +------------+ + | doubled| + +------------+ + |[2, 4, 6, 8]| + +------------+ + + >>> def alternate(x, i): + ... return when(i % 2 == 0, x).otherwise(-x) + ... + >>> df.select(transform("values", alternate).alias("alternated")).show() + +--------------+ + | alternated| + +--------------+ + |[1, -2, 3, -4]| + +--------------+ """ - return _invoke_function("row_number") + return _invoke_higher_order_function("transform", [col], [f]) @_try_remote_functions -def dense_rank() -> Column: +def exists(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: """ - Window function: returns the rank of rows within a window partition, without any gaps. - - The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking - sequence when there are ties. That is, if you were ranking a competition using dense_rank - and had three people tie for second place, you would say that all three were in second - place and that the next person came in third. Rank would give me sequential numbers, making - the person that came in third place (after the ties) would register as coming in fifth. - - This is equivalent to the DENSE_RANK function in SQL. + Returns whether a predicate holds for one or more elements in the array. - .. versionadded:: 1.6.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + ``(x: Column) -> Column: ...`` returning the Boolean expression. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). + Returns ------- :class:`~pyspark.sql.Column` - the column for calculating ranks. - - See Also - -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` + True if "any" element of an array evaluates to True when passed as an argument to + given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("drank", sf.dense_rank().over(w)).show() - +-----+-----+ - |value|drank| - +-----+-----+ - | 1| 1| - | 1| 1| - | 2| 2| - | 3| 3| - | 3| 3| - | 4| 4| - +-----+-----+ + >>> df = spark.createDataFrame([(1, [1, 2, 3, 4]), (2, [3, -1, 0])],("key", "values")) + >>> df.select(exists("values", lambda x: x < 0).alias("any_negative")).show() + +------------+ + |any_negative| + +------------+ + | false| + | true| + +------------+ """ - return _invoke_function("dense_rank") + return _invoke_higher_order_function("exists", [col], [f]) @_try_remote_functions -def rank() -> Column: +def forall(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: """ - Window function: returns the rank of rows within a window partition. + Returns whether a predicate holds for every element in the array. - The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking - sequence when there are ties. That is, if you were ranking a competition using dense_rank - and had three people tie for second place, you would say that all three were in second - place and that the next person came in third. Rank would give me sequential numbers, making - the person that came in third place (after the ties) would register as coming in fifth. - - This is equivalent to the RANK function in SQL. - - .. versionadded:: 1.6.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + ``(x: Column) -> Column: ...`` returning the Boolean expression. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). + Returns ------- :class:`~pyspark.sql.Column` - the column for calculating ranks. - - See Also - -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.row_number` + True if "all" elements of an array evaluates to True when passed as an argument to + given function and False otherwise. + Returns a column that evaluates to a boolean. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("drank", sf.rank().over(w)).show() - +-----+-----+ - |value|drank| - +-----+-----+ - | 1| 1| - | 1| 1| - | 2| 3| - | 3| 4| - | 3| 4| - | 4| 6| - +-----+-----+ + >>> df = spark.createDataFrame( + ... [(1, ["bar"]), (2, ["foo", "bar"]), (3, ["foobar", "foo"])], + ... ("key", "values") + ... ) + >>> df.select(forall("values", lambda x: x.rlike("foo")).alias("all_foo")).show() + +-------+ + |all_foo| + +-------+ + | false| + | false| + | true| + +-------+ """ - return _invoke_function("rank") + return _invoke_higher_order_function("forall", [col], [f]) -@_try_remote_functions -def counter_diff(value: "ColumnOrName", startTime: Optional["ColumnOrName"] = None) -> Column: - """ - Window function: computes the differences between consecutive cumulative counter values in a - time series, thereby converting the counter from the cumulative to the delta format. +@overload +def filter(col: "ColumnOrName", f: Callable[[Column], Column]) -> Column: ... - Gracefully handles counter resets by returning NULL. Counter resets are detected when the - counter value decreases, or when the start time advances between rows. - Use the PARTITION BY clause of the window to separate independent counters. This is done by - specifying all columns which uniquely identify a time series. These are typically the counter - name and any attributes tied to the counter. +@overload +def filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: ... - Use the ORDER BY clause of the window to order the observations by the associated timestamp - in ascending order. - .. versionadded:: 4.3.0 +@_try_remote_functions +def filter( + col: "ColumnOrName", + f: Union[Callable[[Column], Column], Callable[[Column, Column], Column]], +) -> Column: + """ + Returns an array of elements for which a predicate holds in a given array. + + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. Parameters ---------- - value : :class:`~pyspark.sql.Column` or column name - A cumulative counter. Must be a numeric data type. Must be non-negative. - startTime : :class:`~pyspark.sql.Column` or column name, optional - An optional timestamp parameter which indicates when the counter was last set to zero. - Used to signal counter resets. + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + f : function + A function that returns the Boolean expression. + Can take one of the following forms: + + - Unary ``(x: Column) -> Column: ...`` + - Binary ``(x: Column, i: Column) -> Column...``, where the second argument is + a 0-based index of the element. + + and can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - The difference between the current and previous counter value within the window partition. + filtered array of elements where given function evaluated to True + when passed as an argument. + Returns a column that evaluates to an array. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> from datetime import datetime >>> df = spark.createDataFrame( - ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100), - ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200), - ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400), - ... ('http_requests', datetime(2026, 1, 1, 0, 3), 50), - ... ('http_requests', datetime(2026, 1, 1, 0, 4), 100)], - ... "m STRING, t TIMESTAMP_NTZ, c INT") - >>> w = Window.partitionBy("m").orderBy("t") - >>> df.select("m", "t", "c", sf.counter_diff("c").over(w).alias("diff")).show() - +-------------+-------------------+---+----+ - | m| t| c|diff| - +-------------+-------------------+---+----+ - |http_requests|2026-01-01 00:00:00|100|NULL| - |http_requests|2026-01-01 00:01:00|200| 100| - |http_requests|2026-01-01 00:02:00|400| 200| - |http_requests|2026-01-01 00:03:00| 50|NULL| - |http_requests|2026-01-01 00:04:00|100| 50| - +-------------+-------------------+---+----+ - - >>> df2 = spark.createDataFrame( - ... [('http_requests', datetime(2026, 1, 1, 0, 0), 100, datetime(2026, 1, 1, 0, 0)), - ... ('http_requests', datetime(2026, 1, 1, 0, 1), 200, datetime(2026, 1, 1, 0, 0)), - ... ('http_requests', datetime(2026, 1, 1, 0, 2), 400, datetime(2026, 1, 1, 0, 0)), - ... ('http_requests', datetime(2026, 1, 1, 0, 3), 500, datetime(2026, 1, 1, 0, 2, 15)), - ... ('http_requests', datetime(2026, 1, 1, 0, 4), 600, datetime(2026, 1, 1, 0, 2, 15))], - ... "m STRING, t TIMESTAMP_NTZ, c INT, s TIMESTAMP_NTZ") - >>> df2.select("m", "t", "s", "c", sf.counter_diff("c", "s").over(w).alias("diff")).show() - +-------------+-------------------+-------------------+---+----+ - | m| t| s| c|diff| - +-------------+-------------------+-------------------+---+----+ - |http_requests|2026-01-01 00:00:00|2026-01-01 00:00:00|100|NULL| - |http_requests|2026-01-01 00:01:00|2026-01-01 00:00:00|200| 100| - |http_requests|2026-01-01 00:02:00|2026-01-01 00:00:00|400| 200| - |http_requests|2026-01-01 00:03:00|2026-01-01 00:02:15|500|NULL| - |http_requests|2026-01-01 00:04:00|2026-01-01 00:02:15|600| 100| - +-------------+-------------------+-------------------+---+----+ + ... [(1, ["2018-09-20", "2019-02-03", "2019-07-01", "2020-06-01"])], + ... ("key", "values") + ... ) + >>> def after_second_quarter(x): + ... return month(to_date(x)) > 6 + ... + >>> df.select( + ... filter("values", after_second_quarter).alias("after_second_quarter") + ... ).show(truncate=False) + +------------------------+ + |after_second_quarter | + +------------------------+ + |[2018-09-20, 2019-07-01]| + +------------------------+ """ - if startTime is None: - return _invoke_function_over_columns("counter_diff", value) - return _invoke_function_over_columns("counter_diff", value, startTime) + return _invoke_higher_order_function("filter", [col], [f]) @_try_remote_functions -def cume_dist() -> Column: +def aggregate( + col: "ColumnOrName", + initialValue: "ColumnOrName", + merge: Callable[[Column, Column], Column], + finish: Optional[Callable[[Column], Column]] = None, +) -> Column: """ - Window function: returns the cumulative distribution of values within a window partition, - i.e. the fraction of rows that are below the current row. + Applies a binary operator to an initial state and all elements in the array, + and reduces this to a single state. The final state is converted into the final result + by applying a finish function. - .. versionadded:: 1.6.0 + Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). + + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + initialValue : :class:`~pyspark.sql.Column` or str + initial value. Name of column or expression. + A column of any type. + merge : function + a binary function ``(acc: Column, x: Column) -> Column...`` returning expression + of the same type as ``initialValue``. + finish : function, optional + an optional unary function ``(x: Column) -> Column: ...`` + used to convert accumulated value. + Returns ------- :class:`~pyspark.sql.Column` - the column for calculating cumulative distribution. - - See Also - -------- - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` + final value after aggregate function is applied. + Returns a column of the same type as ``initialValue``. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("cd", sf.cume_dist().over(w)).show() - +-----+---+ - |value| cd| - +-----+---+ - | 1|0.2| - | 2|0.4| - | 3|0.8| - | 3|0.8| - | 4|1.0| - +-----+---+ + >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) + >>> df.select(aggregate("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() + +----+ + | sum| + +----+ + |42.0| + +----+ + + >>> def merge(acc, x): + ... count = acc.count + 1 + ... sum = acc.sum + x + ... return struct(count.alias("count"), sum.alias("sum")) + ... + >>> df.select( + ... aggregate( + ... "values", + ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), + ... merge, + ... lambda acc: acc.sum / acc.count, + ... ).alias("mean") + ... ).show() + +----+ + |mean| + +----+ + | 8.4| + +----+ """ - return _invoke_function("cume_dist") + if finish is not None: + return _invoke_higher_order_function("aggregate", [col, initialValue], [merge, finish]) + + else: + return _invoke_higher_order_function("aggregate", [col, initialValue], [merge]) @_try_remote_functions -def percent_rank() -> Column: +def reduce( + col: "ColumnOrName", + initialValue: "ColumnOrName", + merge: Callable[[Column, Column], Column], + finish: Optional[Callable[[Column], Column]] = None, +) -> Column: """ - Window function: returns the relative rank (i.e. percentile) of rows within a window partition. + Applies a binary operator to an initial state and all elements in the array, + and reduces this to a single state. The final state is converted into the final result + by applying a finish function. - .. versionadded:: 1.6.0 + Both functions can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or str + name of column or expression. + A column that evaluates to an array. + initialValue : :class:`~pyspark.sql.Column` or str + initial value. Name of column or expression. + A column of any type. + merge : function + a binary function ``(acc: Column, x: Column) -> Column...`` returning expression + of the same type as ``zero``. + finish : function, optional + an optional unary function ``(x: Column) -> Column: ...`` + used to convert accumulated value. Returns ------- :class:`~pyspark.sql.Column` - the column for calculating relative rank. - - See Also - -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.ntile` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` + final value after aggregate function is applied. + Returns a column of the same type as ``initialValue``. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame([1, 1, 2, 3, 3, 4], "int") - >>> w = Window.orderBy("value") - >>> df.withColumn("pr", sf.percent_rank().over(w)).show() - +-----+---+ - |value| pr| - +-----+---+ - | 1|0.0| - | 1|0.0| - | 2|0.4| - | 3|0.6| - | 3|0.6| - | 4|1.0| - +-----+---+ + >>> df = spark.createDataFrame([(1, [20.0, 4.0, 2.0, 6.0, 10.0])], ("id", "values")) + >>> df.select(reduce("values", lit(0.0), lambda acc, x: acc + x).alias("sum")).show() + +----+ + | sum| + +----+ + |42.0| + +----+ + + >>> def merge(acc, x): + ... count = acc.count + 1 + ... sum = acc.sum + x + ... return struct(count.alias("count"), sum.alias("sum")) + ... + >>> df.select( + ... reduce( + ... "values", + ... struct(lit(0).alias("count"), lit(0.0).alias("sum")), + ... merge, + ... lambda acc: acc.sum / acc.count, + ... ).alias("mean") + ... ).show() + +----+ + |mean| + +----+ + | 8.4| + +----+ """ - return _invoke_function("percent_rank") + if finish is not None: + return _invoke_higher_order_function("reduce", [col, initialValue], [merge, finish]) + + else: + return _invoke_higher_order_function("reduce", [col, initialValue], [merge]) @_try_remote_functions -def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: +def zip_with( + left: "ColumnOrName", + right: "ColumnOrName", + f: Callable[[Column, Column], Column], +) -> Column: """ - Window function: returns the value that is `offset` rows before the current row, and - `default` if there is less than `offset` rows before the current row. For example, - an `offset` of one will return the previous row at any given point in the window partition. - - This is equivalent to the LAG function in SQL. + Merge two given arrays, element-wise, into a single array using a function. + If one array is shorter, nulls are appended at the end to match the length of the longer + array, before applying the function. - .. versionadded:: 1.4.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - name of column or expression - offset : int, optional default 1 - number of row to extend - default : optional - default value + left : :class:`~pyspark.sql.Column` or str + name of the first column or expression. + A column that evaluates to an array. + right : :class:`~pyspark.sql.Column` or str + name of the second column or expression. + A column that evaluates to an array. + f : function + a binary function ``(x1: Column, x2: Column) -> Column...`` + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - value before current row based on `offset`. - - See Also - -------- - :meth:`pyspark.sql.functions.lead` + array of calculated values derived by applying given function to each pair of arguments. + Returns a column that evaluates to an array. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ - - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("previous_value", sf.lag("c2").over(w)).show() - +---+---+--------------+ - | c1| c2|previous_value| - +---+---+--------------+ - | a| 1| NULL| - | a| 2| 1| - | a| 3| 2| - | b| 2| NULL| - | b| 8| 2| - +---+---+--------------+ - - >>> df.withColumn("previous_value", sf.lag("c2", 1, 0).over(w)).show() - +---+---+--------------+ - | c1| c2|previous_value| - +---+---+--------------+ - | a| 1| 0| - | a| 2| 1| - | a| 3| 2| - | b| 2| 0| - | b| 8| 2| - +---+---+--------------+ + >>> df = spark.createDataFrame([(1, [1, 3, 5, 8], [0, 2, 4, 6])], ("id", "xs", "ys")) + >>> df.select(zip_with("xs", "ys", lambda x, y: x ** y).alias("powers")).show(truncate=False) + +---------------------------+ + |powers | + +---------------------------+ + |[1.0, 9.0, 625.0, 262144.0]| + +---------------------------+ - >>> df.withColumn("previous_value", sf.lag("c2", 2, -1).over(w)).show() - +---+---+--------------+ - | c1| c2|previous_value| - +---+---+--------------+ - | a| 1| -1| - | a| 2| -1| - | a| 3| 1| - | b| 2| -1| - | b| 8| -1| - +---+---+--------------+ + >>> df = spark.createDataFrame([(1, ["foo", "bar"], [1, 2, 3])], ("id", "xs", "ys")) + >>> df.select(zip_with("xs", "ys", lambda x, y: concat_ws("_", x, y)).alias("xs_ys")).show() + +-----------------+ + | xs_ys| + +-----------------+ + |[foo_1, bar_2, 3]| + +-----------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "lag", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) - ) + return _invoke_higher_order_function("zip_with", [left, right], [f]) @_try_remote_functions -def lead(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: +def transform_keys(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: """ - Window function: returns the value that is `offset` rows after the current row, and - `default` if there is less than `offset` rows after the current row. For example, - an `offset` of one will return the next row at any given point in the window partition. - - This is equivalent to the LEAD function in SQL. + Applies a function to every key-value pair in a map and returns + a map with the results of those applications as the new keys for the pairs. - .. versionadded:: 1.4.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + col : :class:`~pyspark.sql.Column` or str name of column or expression - offset : int, optional default 1 - number of row to extend - default : optional - default value + f : function + a binary function ``(k: Column, v: Column) -> Column...`` + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - value after current row based on `offset`. - - See Also - -------- - :meth:`pyspark.sql.functions.lag` + a new map of entries where new keys were calculated by applying given function to + each key value argument. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ - - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("next_value", sf.lead("c2").over(w)).show() - +---+---+----------+ - | c1| c2|next_value| - +---+---+----------+ - | a| 1| 2| - | a| 2| 3| - | a| 3| NULL| - | b| 2| 8| - | b| 8| NULL| - +---+---+----------+ - - >>> df.withColumn("next_value", sf.lead("c2", 1, 0).over(w)).show() - +---+---+----------+ - | c1| c2|next_value| - +---+---+----------+ - | a| 1| 2| - | a| 2| 3| - | a| 3| 0| - | b| 2| 8| - | b| 8| 0| - +---+---+----------+ - - >>> df.withColumn("next_value", sf.lead("c2", 2, -1).over(w)).show() - +---+---+----------+ - | c1| c2|next_value| - +---+---+----------+ - | a| 1| 3| - | a| 2| -1| - | a| 3| -1| - | b| 2| -1| - | b| 8| -1| - +---+---+----------+ - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "lead", _to_java_column(col), _enum_to_value(offset), _enum_to_value(default) - ) + >>> df = spark.createDataFrame([(1, {"foo": -2.0, "bar": 2.0})], ("id", "data")) + >>> row = df.select(transform_keys( + ... "data", lambda k, _: upper(k)).alias("data_upper") + ... ).head() + >>> sorted(row["data_upper"].items()) + [('BAR', 2.0), ('FOO', -2.0)] + """ + return _invoke_higher_order_function("transform_keys", [col], [f]) @_try_remote_functions -def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = False) -> Column: +def transform_values(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: """ - Window function: returns the value that is the `offset`\\th row of the window frame - (counting from 1), and `null` if the size of window frame is less than `offset` rows. - - It will return the `offset`\\th non-null value it sees when `ignoreNulls` is set to - true. If all values are null, then null is returned. - - This is equivalent to the nth_value function in SQL. + Applies a function to every key-value pair in a map and returns + a map with the results of those applications as the new values for the pairs. .. versionadded:: 3.1.0 @@ -26227,732 +26580,305 @@ def nth_value(col: "ColumnOrName", offset: int, ignoreNulls: Optional[bool] = Fa Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + col : :class:`~pyspark.sql.Column` or str name of column or expression - offset : int - number of row to use as the value - ignoreNulls : bool, optional - indicates the Nth value should skip null in the - determination of which row to use + f : function + a binary function ``(k: Column, v: Column) -> Column...`` + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - value of nth row. - - See Also - -------- - :meth:`pyspark.sql.functions.first_value` - :meth:`pyspark.sql.functions.last_value` + a new map of entries where new values were calculated by applying given function to + each key value argument. Examples -------- - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ - - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("nth_value", sf.nth_value("c2", 1).over(w)).show() - +---+---+---------+ - | c1| c2|nth_value| - +---+---+---------+ - | a| 1| 1| - | a| 2| 1| - | a| 3| 1| - | b| 2| 2| - | b| 8| 2| - +---+---+---------+ - - >>> df.withColumn("nth_value", sf.nth_value("c2", 2).over(w)).show() - +---+---+---------+ - | c1| c2|nth_value| - +---+---+---------+ - | a| 1| NULL| - | a| 2| 2| - | a| 3| 2| - | b| 2| NULL| - | b| 8| 8| - +---+---+---------+ + >>> df = spark.createDataFrame([(1, {"IT": 10.0, "SALES": 2.0, "OPS": 24.0})], ("id", "data")) + >>> row = df.select(transform_values( + ... "data", lambda k, v: when(k.isin("IT", "OPS"), v + 10.0).otherwise(v) + ... ).alias("new_data")).head() + >>> sorted(row["new_data"].items()) + [('IT', 20.0), ('OPS', 34.0), ('SALES', 2.0)] """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "nth_value", _to_java_column(col), _enum_to_value(offset), _enum_to_value(ignoreNulls) - ) + return _invoke_higher_order_function("transform_values", [col], [f]) @_try_remote_functions -def ntile(n: int) -> Column: +def map_filter(col: "ColumnOrName", f: Callable[[Column, Column], Column]) -> Column: """ - Window function: returns the ntile group id (from 1 to `n` inclusive) - in an ordered window partition. For example, if `n` is 4, the first - quarter of the rows will get value 1, the second quarter will get 2, - the third quarter will get 3, and the last quarter will get 4. - - This is equivalent to the NTILE function in SQL. + Collection function: Returns a new map column whose key-value pairs satisfy a given + predicate function. - .. versionadded:: 1.4.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - n : int - an integer + col : :class:`~pyspark.sql.Column` or str + The name of the column or a column expression representing the map to be filtered. + f : function + A binary function ``(k: Column, v: Column) -> Column...`` that defines the predicate. + This function should return a boolean column that will be used to filter the input map. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - portioned group id. - - See Also - -------- - :meth:`pyspark.sql.functions.cume_dist` - :meth:`pyspark.sql.functions.dense_rank` - :meth:`pyspark.sql.functions.percent_rank` - :meth:`pyspark.sql.functions.rank` - :meth:`pyspark.sql.functions.row_number` + A new map column containing only the key-value pairs that satisfy the predicate. Examples -------- + Example 1: Filtering a map with a simple condition + >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Window - >>> df = spark.createDataFrame( - ... [("a", 1), ("a", 2), ("a", 3), ("b", 8), ("b", 2)], ["c1", "c2"]) - >>> df.show() - +---+---+ - | c1| c2| - +---+---+ - | a| 1| - | a| 2| - | a| 3| - | b| 8| - | b| 2| - +---+---+ + >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) + >>> row = df.select( + ... sf.map_filter("data", lambda _, v: v > 30.0).alias("data_filtered") + ... ).head() + >>> sorted(row["data_filtered"].items()) + [('baz', 32.0), ('foo', 42.0)] - >>> w = Window.partitionBy("c1").orderBy("c2") - >>> df.withColumn("ntile", sf.ntile(2).over(w)).show() - +---+---+-----+ - | c1| c2|ntile| - +---+---+-----+ - | a| 1| 1| - | a| 2| 1| - | a| 3| 2| - | b| 2| 1| - | b| 8| 2| - +---+---+-----+ - """ - return _invoke_function("ntile", int(_enum_to_value(n))) + Example 2: Filtering a map with a condition on keys + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) + >>> row = df.select( + ... sf.map_filter("data", lambda k, _: k.startswith("b")).alias("data_filtered") + ... ).head() + >>> sorted(row["data_filtered"].items()) + [('bar', 1.0), ('baz', 32.0)] + Example 3: Filtering a map with a complex condition -# ---------------------- Generator Functions ---------------------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, {"foo": 42.0, "bar": 1.0, "baz": 32.0})], ("id", "data")) + >>> row = df.select( + ... sf.map_filter("data", lambda k, v: k.startswith("b") & (v > 1.0)).alias("data_filtered") + ... ).head() + >>> sorted(row["data_filtered"].items()) + [('baz', 32.0)] + """ + return _invoke_higher_order_function("map_filter", [col], [f]) @_try_remote_functions -def explode(col: "ColumnOrName") -> Column: +def map_zip_with( + col1: "ColumnOrName", + col2: "ColumnOrName", + f: Callable[[Column, Column, Column], Column], +) -> Column: """ - Returns a new row for each element in the given array or map. - Uses the default column name `col` for elements in the array and - `key` and `value` for elements in the map unless specified otherwise. + Collection: Merges two given maps into a single map by applying a function to + the key-value pairs. - .. versionadded:: 1.4.0 + .. versionadded:: 3.1.0 .. versionchanged:: 3.4.0 Supports Spark Connect. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Target column to work on. - A column that evaluates to an array or map. + col1 : :class:`~pyspark.sql.Column` or str + The name of the first column or a column expression representing the first map. + col2 : :class:`~pyspark.sql.Column` or str + The name of the second column or a column expression representing the second map. + f : function + A ternary function ``(k: Column, v1: Column, v2: Column) -> Column...`` that defines + how to merge the values from the two maps. This function should return a column that + will be used as the value in the resulting map. + Can use methods of :class:`~pyspark.sql.Column`, functions defined in + :py:mod:`pyspark.sql.functions` and Scala ``UserDefinedFunctions``. + Python ``UserDefinedFunctions`` are not supported + (`SPARK-27052 `__). Returns ------- :class:`~pyspark.sql.Column` - One row per array item or map key value. - Returns a column of the element type of the input array, or the key and value - columns of the input map. - - See Also - -------- - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` - - Notes - ----- - Only one explode is allowed per SELECT clause. + A new map column where each key-value pair is the result of applying the function to + the corresponding key-value pairs in the input maps. Examples -------- - Example 1: Exploding an array column + Example 1: Merging two maps with a simple function >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.show() - +---+---------------+ - | i| a| - +---+---------------+ - | 1|[1, 2, 3, NULL]| - | 2| []| - | 3| NULL| - +---+---------------+ - - >>> df.select('*', sf.explode('a')).show() - +---+---------------+----+ - | i| a| col| - +---+---------------+----+ - | 1|[1, 2, 3, NULL]| 1| - | 1|[1, 2, 3, NULL]| 2| - | 1|[1, 2, 3, NULL]| 3| - | 1|[1, 2, 3, NULL]|NULL| - +---+---------------+----+ + >>> df = spark.createDataFrame([ + ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], + ... ("id", "map1", "map2")) + >>> row = df.select( + ... sf.map_zip_with("map1", "map2", lambda _, v1, v2: v1 + v2).alias("updated_data") + ... ).head() + >>> sorted(row["updated_data"].items()) + [('A', 4), ('B', 6)] - Example 2: Exploding a map column + Example 2: Merging two maps with a complex function >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.show(truncate=False) - +---+---------------------------+ - |i |m | - +---+---------------------------+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| - |2 |{} | - |3 |NULL | - +---+---------------------------+ - - >>> df.select('*', sf.explode('m')).show(truncate=False) - +---+---------------------------+---+-----+ - |i |m |key|value| - +---+---------------------------+---+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | - +---+---------------------------+---+-----+ - - Example 3: Exploding multiple array columns - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(1,2) AS a1, ARRAY(3,4,5) AS a2') - >>> df.select( - ... '*', sf.explode('a1').alias('v1') - ... ).select('*', sf.explode('a2').alias('v2')).show() - +------+---------+---+---+ - | a1| a2| v1| v2| - +------+---------+---+---+ - |[1, 2]|[3, 4, 5]| 1| 3| - |[1, 2]|[3, 4, 5]| 1| 4| - |[1, 2]|[3, 4, 5]| 1| 5| - |[1, 2]|[3, 4, 5]| 2| 3| - |[1, 2]|[3, 4, 5]| 2| 4| - |[1, 2]|[3, 4, 5]| 2| 5| - +------+---------+---+---+ + >>> df = spark.createDataFrame([ + ... (1, {"A": 1, "B": 2}, {"A": 3, "B": 4})], + ... ("id", "map1", "map2")) + >>> row = df.select( + ... sf.map_zip_with("map1", "map2", + ... lambda k, v1, v2: sf.when(k == "A", v1 + v2).otherwise(v1 - v2) + ... ).alias("updated_data") + ... ).head() + >>> sorted(row["updated_data"].items()) + [('A', 4), ('B', -2)] - Example 4: Exploding an array of struct column + Example 3: Merging two maps with mismatched keys - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select(sf.explode('a').alias("s")).select("s.*").show() - +---+---+ - | a| b| - +---+---+ - | 1| 2| - | 3| 4| - +---+---+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([ + ... (1, {"A": 1, "B": 2}, {"B": 3, "C": 4})], + ... ("id", "map1", "map2")) + >>> row = df.select( + ... sf.map_zip_with("map1", "map2", + ... lambda _, v1, v2: sf.when(v2.isNull(), v1).otherwise(v1 + v2) + ... ).alias("updated_data") + ... ).head() + >>> sorted(row["updated_data"].items()) + [('A', 1), ('B', 5), ('C', None)] """ - return _invoke_function_over_columns("explode", col) + return _invoke_higher_order_function("map_zip_with", [col1, col2], [f]) @_try_remote_functions -def posexplode(col: "ColumnOrName") -> Column: +def str_to_map( + text: "ColumnOrName", + pairDelim: Optional["ColumnOrName"] = None, + keyValueDelim: Optional["ColumnOrName"] = None, +) -> Column: """ - Returns a new row for each element with position in the given array or map. - Uses the default column name `pos` for position, and `col` for elements in the - array and `key` and `value` for elements in the map unless specified otherwise. - - .. versionadded:: 2.1.0 + Map function: Converts a string into a map after splitting the text into key/value pairs + using delimiters. Both `pairDelim` and `keyValueDelim` are treated as regular expressions. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. + text : :class:`~pyspark.sql.Column` or str + Input column or strings. + A column that evaluates to a string. + pairDelim : :class:`~pyspark.sql.Column` or str, optional + Delimiter to use to split pairs. Default is comma (,). + A column that evaluates to a string. + keyValueDelim : :class:`~pyspark.sql.Column` or str, optional + Delimiter to use to split key/value. Default is colon (:). + A column that evaluates to a string. Returns ------- :class:`~pyspark.sql.Column` - one row per array item or map key value including positions as a separate column. - - See Also - -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` + A new column of map type where each string in the original column is converted into a map. + Returns a column that evaluates to a map. Examples -------- - Example 1: Exploding an array column + Example 1: Using default delimiters >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.show() - +---+---------------+ - | i| a| - +---+---------------+ - | 1|[1, 2, 3, NULL]| - | 2| []| - | 3| NULL| - +---+---------------+ + >>> df = spark.createDataFrame([("a:1,b:2,c:3",)], ["e"]) + >>> df.select(sf.str_to_map(df.e)).show(truncate=False) + +------------------------+ + |str_to_map(e, ,, :) | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + +------------------------+ - >>> df.select('*', sf.posexplode('a')).show() - +---+---------------+---+----+ - | i| a|pos| col| - +---+---------------+---+----+ - | 1|[1, 2, 3, NULL]| 0| 1| - | 1|[1, 2, 3, NULL]| 1| 2| - | 1|[1, 2, 3, NULL]| 2| 3| - | 1|[1, 2, 3, NULL]| 3|NULL| - +---+---------------+---+----+ + Example 2: Using custom delimiters - Example 2: Exploding a map column + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("a=1;b=2;c=3",)], ["e"]) + >>> df.select(sf.str_to_map(df.e, sf.lit(";"), sf.lit("="))).show(truncate=False) + +------------------------+ + |str_to_map(e, ;, =) | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + +------------------------+ + + Example 3: Using different delimiters for different rows >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.show(truncate=False) - +---+---------------------------+ - |i |m | - +---+---------------------------+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}| - |2 |{} | - |3 |NULL | - +---+---------------------------+ + >>> df = spark.createDataFrame([("a:1,b:2,c:3",), ("d=4;e=5;f=6",)], ["e"]) + >>> df.select(sf.str_to_map(df.e, + ... sf.when(df.e.contains(";"), sf.lit(";")).otherwise(sf.lit(",")), + ... sf.when(df.e.contains("="), sf.lit("=")).otherwise(sf.lit(":"))).alias("str_to_map") + ... ).show(truncate=False) + +------------------------+ + |str_to_map | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + |{d -> 4, e -> 5, f -> 6}| + +------------------------+ - >>> df.select('*', sf.posexplode('m')).show(truncate=False) - +---+---------------------------+---+---+-----+ - |i |m |pos|key|value| - +---+---------------------------+---+---+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | - +---+---------------------------+---+---+-----+ + Example 4: Using a column of delimiters + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("a:1,b:2,c:3", ","), ("d=4;e=5;f=6", ";")], ["e", "delim"]) + >>> df.select(sf.str_to_map(df.e, df.delim, sf.lit(":"))).show(truncate=False) + +---------------------------------------+ + |str_to_map(e, delim, :) | + +---------------------------------------+ + |{a -> 1, b -> 2, c -> 3} | + |{d=4 -> NULL, e=5 -> NULL, f=6 -> NULL}| + +---------------------------------------+ + + Example 5: Using a column of key/value delimiters + + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("a:1,b:2,c:3", ":"), ("d=4;e=5;f=6", "=")], ["e", "delim"]) + >>> df.select(sf.str_to_map(df.e, sf.lit(","), df.delim)).show(truncate=False) + +------------------------+ + |str_to_map(e, ,, delim) | + +------------------------+ + |{a -> 1, b -> 2, c -> 3}| + |{d -> 4;e=5;f=6} | + +------------------------+ """ - return _invoke_function_over_columns("posexplode", col) + if pairDelim is None: + pairDelim = lit(",") + if keyValueDelim is None: + keyValueDelim = lit(":") + return _invoke_function_over_columns("str_to_map", text, pairDelim, keyValueDelim) + + +# ---------------------- Partition transform functions -------------------------------- @_try_remote_functions -def inline(col: "ColumnOrName") -> Column: +def years(col: "ColumnOrName") -> Column: """ - Explodes an array of structs into a table. + Partition transform function: A transform for timestamps and dates + to partition data into years. - This function takes an input column containing an array of structs and returns a - new column where each struct in the array is exploded into a separate row. + .. versionadded:: 3.1.0 - .. versionadded:: 3.4.0 + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 4.0.0 + Use :func:`partitioning.years` instead. Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - Input column of values to explode. + col : :class:`~pyspark.sql.Column` or str + target date or timestamp column to work on. Returns ------- :class:`~pyspark.sql.Column` - Generator expression with the inline exploded result. + data partitioned by years. - See Also - -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline_outer` - - Examples - -------- - Example 1: Using inline with a single struct array column - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select('*', sf.inline(df.a)).show() - +----------------+---+---+ - | a| a| b| - +----------------+---+---+ - |[{1, 2}, {3, 4}]| 1| 2| - |[{1, 2}, {3, 4}]| 3| 4| - +----------------+---+---+ - - Example 2: Using inline with a column name - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select('*', sf.inline('a')).show() - +----------------+---+---+ - | a| a| b| - +----------------+---+---+ - |[{1, 2}, {3, 4}]| 1| 2| - |[{1, 2}, {3, 4}]| 3| 4| - +----------------+---+---+ - - Example 3: Using inline with an alias - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a') - >>> df.select('*', sf.inline('a').alias("c1", "c2")).show() - +----------------+---+---+ - | a| c1| c2| - +----------------+---+---+ - |[{1, 2}, {3, 4}]| 1| 2| - |[{1, 2}, {3, 4}]| 3| 4| - +----------------+---+---+ - - Example 4: Using inline with multiple struct array columns - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT ARRAY(NAMED_STRUCT("a",1,"b",2), NAMED_STRUCT("a",3,"b",4)) AS a1, ARRAY(NAMED_STRUCT("c",5,"d",6), NAMED_STRUCT("c",7,"d",8)) AS a2') - >>> df.select( - ... '*', sf.inline('a1') - ... ).select('*', sf.inline('a2')).show() - +----------------+----------------+---+---+---+---+ - | a1| a2| a| b| c| d| - +----------------+----------------+---+---+---+---+ - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 5| 6| - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 1| 2| 7| 8| - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 5| 6| - |[{1, 2}, {3, 4}]|[{5, 6}, {7, 8}]| 3| 4| 7| 8| - +----------------+----------------+---+---+---+---+ - - Example 5: Using inline with a nested struct array column - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql('SELECT NAMED_STRUCT("a",1,"b",2,"c",ARRAY(NAMED_STRUCT("c",3,"d",4), NAMED_STRUCT("c",5,"d",6))) AS s') - >>> df.select('*', sf.inline('s.c')).show(truncate=False) - +------------------------+---+---+ - |s |c |d | - +------------------------+---+---+ - |{1, 2, [{3, 4}, {5, 6}]}|3 |4 | - |{1, 2, [{3, 4}, {5, 6}]}|5 |6 | - +------------------------+---+---+ - - Example 6: Using inline with a column containing: array continaing null, empty array and null - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') - >>> df.show(truncate=False) - +---+----------------------+ - |i |s | - +---+----------------------+ - |1 |[{1, 2}, NULL, {3, 4}]| - |2 |[] | - |3 |NULL | - +---+----------------------+ - - >>> df.select('*', sf.inline('s')).show(truncate=False) - +---+----------------------+----+----+ - |i |s |a |b | - +---+----------------------+----+----+ - |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | - |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| - |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | - +---+----------------------+----+----+ - """ - return _invoke_function_over_columns("inline", col) - - -@_try_remote_functions -def explode_outer(col: "ColumnOrName") -> Column: - """ - Returns a new row for each element in the given array or map. - Unlike explode, if the array/map is null or empty then null is produced. - Uses the default column name `col` for elements in the array and - `key` and `value` for elements in the map unless specified otherwise. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - A column that evaluates to an array or map. - - Returns - ------- - :class:`~pyspark.sql.Column` - one row per array item or map key value. - Returns a column of the element type of the input array, or the key and value - columns of the input map. - - See Also - -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` - - Examples - -------- - Example 1: Using an array column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.select('*', sf.explode_outer('a')).show() - +---+---------------+----+ - | i| a| col| - +---+---------------+----+ - | 1|[1, 2, 3, NULL]| 1| - | 1|[1, 2, 3, NULL]| 2| - | 1|[1, 2, 3, NULL]| 3| - | 1|[1, 2, 3, NULL]|NULL| - | 2| []|NULL| - | 3| NULL|NULL| - +---+---------------+----+ - - Example 2: Using a map column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.select('*', sf.explode_outer('m')).show(truncate=False) - +---+---------------------------+----+-----+ - |i |m |key |value| - +---+---------------------------+----+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|5 |NULL | - |2 |{} |NULL|NULL | - |3 |NULL |NULL|NULL | - +---+---------------------------+----+-----+ - """ - return _invoke_function_over_columns("explode_outer", col) - - -@_try_remote_functions -def posexplode_outer(col: "ColumnOrName") -> Column: - """ - Returns a new row for each element with position in the given array or map. - Unlike posexplode, if the array/map is null or empty then the row (null, null) is produced. - Uses the default column name `pos` for position, and `col` for elements in the - array and `key` and `value` for elements in the map unless specified otherwise. - - .. versionadded:: 2.3.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - target column to work on. - - Returns - ------- - :class:`~pyspark.sql.Column` - one row per array item or map key value including positions as a separate column. - - See Also - -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.inline` - :meth:`pyspark.sql.functions.inline_outer` - - Examples - -------- - Example 1: Using an array column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(1,2,3,NULL)), (2,ARRAY()), (3,NULL) AS t(i,a)') - >>> df.select('*', sf.posexplode_outer('a')).show() - +---+---------------+----+----+ - | i| a| pos| col| - +---+---------------+----+----+ - | 1|[1, 2, 3, NULL]| 0| 1| - | 1|[1, 2, 3, NULL]| 1| 2| - | 1|[1, 2, 3, NULL]| 2| 3| - | 1|[1, 2, 3, NULL]| 3|NULL| - | 2| []|NULL|NULL| - | 3| NULL|NULL|NULL| - +---+---------------+----+----+ - - Example 2: Using a map column - - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,MAP(1,2,3,4,5,NULL)), (2,MAP()), (3,NULL) AS t(i,m)') - >>> df.select('*', sf.posexplode_outer('m')).show(truncate=False) - +---+---------------------------+----+----+-----+ - |i |m |pos |key |value| - +---+---------------------------+----+----+-----+ - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|0 |1 |2 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|1 |3 |4 | - |1 |{1 -> 2, 3 -> 4, 5 -> NULL}|2 |5 |NULL | - |2 |{} |NULL|NULL|NULL | - |3 |NULL |NULL|NULL|NULL | - +---+---------------------------+----+----+-----+ - """ - return _invoke_function_over_columns("posexplode_outer", col) - - -@_try_remote_functions -def inline_outer(col: "ColumnOrName") -> Column: - """ - Explodes an array of structs into a table. - Unlike inline, if the array is null or empty then null is produced for each nested column. - - .. versionadded:: 3.4.0 - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or column name - input column of values to explode. - - Returns - ------- - :class:`~pyspark.sql.Column` - generator expression with the inline exploded result. - - See Also - -------- - :meth:`pyspark.sql.functions.explode` - :meth:`pyspark.sql.functions.explode_outer` - :meth:`pyspark.sql.functions.posexplode` - :meth:`pyspark.sql.functions.posexplode_outer` - :meth:`pyspark.sql.functions.inline` - - Notes - ----- - Supports Spark Connect. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql('SELECT * FROM VALUES (1,ARRAY(NAMED_STRUCT("a",1,"b",2), NULL, NAMED_STRUCT("a",3,"b",4))), (2,ARRAY()), (3,NULL) AS t(i,s)') - >>> df.printSchema() - root - |-- i: integer (nullable = false) - |-- s: array (nullable = true) - | |-- element: struct (containsNull = true) - | | |-- a: integer (nullable = false) - | | |-- b: integer (nullable = false) - - >>> df.select('*', sf.inline_outer('s')).show(truncate=False) - +---+----------------------+----+----+ - |i |s |a |b | - +---+----------------------+----+----+ - |1 |[{1, 2}, NULL, {3, 4}]|1 |2 | - |1 |[{1, 2}, NULL, {3, 4}]|NULL|NULL| - |1 |[{1, 2}, NULL, {3, 4}]|3 |4 | - |2 |[] |NULL|NULL| - |3 |NULL |NULL|NULL| - +---+----------------------+----+----+ - """ - return _invoke_function_over_columns("inline_outer", col) - - -@_try_remote_functions -def stack(*cols: "ColumnOrName") -> Column: - """ - Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default - unless specified otherwise. - - .. versionadded:: 3.5.0 - - Parameters - ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a literal int for the number of rows to be separated, - and the remaining are input elements to be separated. - - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) - >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c')).show() - +---+---+---+----+----+ - | a| b| c|col0|col1| - +---+---+---+----+----+ - | 1| 2| 3| 1| 2| - | 1| 2| 3| 3|NULL| - +---+---+---+----+----+ - - >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c').alias('x', 'y')).show() - +---+---+---+---+----+ - | a| b| c| x| y| - +---+---+---+---+----+ - | 1| 2| 3| 1| 2| - | 1| 2| 3| 3|NULL| - +---+---+---+---+----+ - - >>> df.select('*', sf.stack(sf.lit(3), df.a, df.b, 'c')).show() - +---+---+---+----+ - | a| b| c|col0| - +---+---+---+----+ - | 1| 2| 3| 1| - | 1| 2| 3| 2| - | 1| 2| 3| 3| - +---+---+---+----+ - - >>> df.select('*', sf.stack(sf.lit(4), df.a, df.b, 'c')).show() - +---+---+---+----+ - | a| b| c|col0| - +---+---+---+----+ - | 1| 2| 3| 1| - | 1| 2| 3| 2| - | 1| 2| 3| 3| - | 1| 2| 3|NULL| - +---+---+---+----+ - """ - return _invoke_function_over_seq_of_columns("stack", cols) - - -# ---------------------- Partition Transformation Functions ---------------------- - - -@_try_remote_functions -def years(col: "ColumnOrName") -> Column: - """ - Partition transform function: A transform for timestamps and dates - to partition data into years. - - .. versionadded:: 3.1.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. deprecated:: 4.0.0 - Use :func:`partitioning.years` instead. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - target date or timestamp column to work on. - - Returns - ------- - :class:`~pyspark.sql.Column` - data partitioned by years. - - Examples + Examples -------- >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP ... years("ts") @@ -27105,4318 +27031,3846 @@ def hours(col: "ColumnOrName") -> Column: @_try_remote_functions -def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column: +def convert_timezone( + sourceTz: Optional[Column], targetTz: Column, sourceTs: "ColumnOrName" +) -> Column: """ - Partition transform function: A transform for any type that partitions - by a hash of the input column. - - .. versionadded:: 3.1.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - .. deprecated:: 4.0.0 - Use :func:`partitioning.bucket` instead. + Converts the timestamp without time zone `sourceTs` + from the `sourceTz` time zone to `targetTz`. - Examples - -------- - >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP - ... bucket(42, "ts") - ... ).createOrReplace() + .. versionadded:: 3.5.0 Parameters ---------- - numBuckets : :class:`~pyspark.sql.Column` or int - the number of buckets - col : :class:`~pyspark.sql.Column` or str - target date or timestamp column to work on. + sourceTz : :class:`~pyspark.sql.Column`, optional + The time zone for the input timestamp. If it is missed, + the current session time zone is used as the source time zone. + A column that evaluates to a string. + targetTz : :class:`~pyspark.sql.Column` + The time zone to which the input timestamp should be converted. + A column that evaluates to a string. + sourceTs : :class:`~pyspark.sql.Column` or column name + A timestamp without time zone. + A column that evaluates to a timestamp. Returns ------- :class:`~pyspark.sql.Column` - data partitioned by given columns. + A new column that contains a timestamp for converted time zone. + Returns a column that evaluates to a timestamp. - Notes - ----- - This function can be used only in combination with - :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` - method of the `DataFrameWriterV2`. + See Also + -------- + :meth:`pyspark.sql.functions.current_timezone` - """ - from pyspark.sql.functions import partitioning + Examples + -------- + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - warnings.warn("Deprecated in 4.0.0, use partitioning.bucket instead.", FutureWarning) + Example 1: Converts the timestamp without time zone `sourceTs`. - return partitioning.bucket(numBuckets, col) + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('2015-04-08 00:00:00',)], ['ts']) + >>> df.select( + ... '*', + ... sf.convert_timezone(None, sf.lit('Asia/Hong_Kong'), 'ts') + ... ).show() # doctest: +SKIP + +-------------------+--------------------------------------------------------+ + | ts|convert_timezone(current_timezone(), Asia/Hong_Kong, ts)| + +-------------------+--------------------------------------------------------+ + |2015-04-08 00:00:00| 2015-04-08 15:00:00| + +-------------------+--------------------------------------------------------+ + + Example 2: Converts the timestamp with time zone `sourceTs`. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('2015-04-08 15:00:00',)], ['ts']) + >>> df.select( + ... '*', + ... sf.convert_timezone(sf.lit('Asia/Hong_Kong'), sf.lit('America/Los_Angeles'), df.ts) + ... ).show() + +-------------------+---------------------------------------------------------+ + | ts|convert_timezone(Asia/Hong_Kong, America/Los_Angeles, ts)| + +-------------------+---------------------------------------------------------+ + |2015-04-08 15:00:00| 2015-04-08 00:00:00| + +-------------------+---------------------------------------------------------+ -# ---------------------- CSV Functions ---------------------- + >>> spark.conf.unset("spark.sql.session.timeZone") + """ + if sourceTz is None: + return _invoke_function_over_columns("convert_timezone", targetTz, sourceTs) + else: + return _invoke_function_over_columns("convert_timezone", sourceTz, targetTz, sourceTs) @_try_remote_functions -def schema_of_csv(csv: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: +def make_dt_interval( + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, +) -> Column: """ - CSV Function: Parses a CSV string and infers its schema in DDL format. - - .. versionadded:: 3.0.0 + Make DayTimeIntervalType duration from days, hours, mins and secs. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - csv : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - A CSV string or a foldable string column containing a CSV string. - options : dict, optional - Options to control parsing. Accepts the same options as the CSV datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + days : :class:`~pyspark.sql.Column` or column name, optional + The number of days, positive or negative. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The number of hours, positive or negative. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The number of minutes, positive or negative. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` - A string representation of a :class:`StructType` parsed from the given CSV. - Returns a column that evaluates to a string. + A new column that contains a DayTimeIntervalType duration. + Returns a column that evaluates to an interval. - Examples + See Also -------- - Example 1: Inferring the schema of a CSV string with different data types + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.make_ym_interval` + :meth:`pyspark.sql.functions.try_make_interval` - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('1|a|true'), {'sep':'|'})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv(1|a|true) | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ + Examples + -------- + Example 1: Make DayTimeIntervalType duration from days, hours, mins and secs. - Example 2: Inferring the schema of a CSV string with missing values + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval(df.day, df.hour, df.min, df.sec)).show(truncate=False) + +---+----+---+--------+------------------------------------------+ + |day|hour|min|sec |make_dt_interval(day, hour, min, sec) | + +---+----+---+--------+------------------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 12:30:01.001001' DAY TO SECOND| + +---+----+---+--------+------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('1||true'), {'sep':'|'})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv(1||true) | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ + Example 2: Make DayTimeIntervalType duration from days, hours and mins. - Example 3: Inferring the schema of a CSV string with a different delimiter + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval(df.day, 'hour', df.min)).show(truncate=False) + +---+----+---+--------+-----------------------------------+ + |day|hour|min|sec |make_dt_interval(day, hour, min, 0)| + +---+----+---+--------+-----------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 12:30:00' DAY TO SECOND| + +---+----+---+--------+-----------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('1;a;true'), {'sep':';'})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv(1;a;true) | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ + Example 3: Make DayTimeIntervalType duration from days and hours. - Example 4: Inferring the schema of a CSV string with quoted fields + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval(df.day, df.hour)).show(truncate=False) + +---+----+---+--------+-----------------------------------+ + |day|hour|min|sec |make_dt_interval(day, hour, 0, 0) | + +---+----+---+--------+-----------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 12:00:00' DAY TO SECOND| + +---+----+---+--------+-----------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_csv(sf.lit('"1","a","true"'), {'sep':','})).show(truncate=False) - +-------------------------------------------+ - |schema_of_csv("1","a","true") | - +-------------------------------------------+ - |STRUCT<_c0: INT, _c1: STRING, _c2: BOOLEAN>| - +-------------------------------------------+ - """ - from pyspark.sql.classic.column import _to_java_column + Example 4: Make DayTimeIntervalType duration from days. - csv = _enum_to_value(csv) - if not isinstance(csv, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "csv", - "arg_type": type(csv).__name__, - }, - ) + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[1, 12, 30, 01.001001]], ['day', 'hour', 'min', 'sec']) + >>> df.select('*', sf.make_dt_interval('day')).show(truncate=False) + +---+----+---+--------+-----------------------------------+ + |day|hour|min|sec |make_dt_interval(day, 0, 0, 0) | + +---+----+---+--------+-----------------------------------+ + |1 |12 |30 |1.001001|INTERVAL '1 00:00:00' DAY TO SECOND| + +---+----+---+--------+-----------------------------------+ - return _invoke_function("schema_of_csv", _to_java_column(lit(csv)), _options_to_str(options)) + Example 5: Make empty interval. + + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.make_dt_interval()).show(truncate=False) + +-----------------------------------+ + |make_dt_interval(0, 0, 0, 0) | + +-----------------------------------+ + |INTERVAL '0 00:00:00' DAY TO SECOND| + +-----------------------------------+ + """ + _days = lit(0) if days is None else days + _hours = lit(0) if hours is None else hours + _mins = lit(0) if mins is None else mins + _secs = lit(decimal.Decimal(0)) if secs is None else secs + return _invoke_function_over_columns("make_dt_interval", _days, _hours, _mins, _secs) @_try_remote_functions -def to_csv(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: +def try_make_interval( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + weeks: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, +) -> Column: """ - CSV Function: Converts a column containing a :class:`StructType` into a CSV string. - Throws an exception, in the case of an unsupported type. - - .. versionadded:: 3.0.0 + This is a special version of `make_interval` that performs the same operation, but returns a + NULL value instead of raising an error if interval cannot be created. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.0.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a struct, array, map, or variant. - Name of column containing a struct. - options: dict, optional - Options to control converting. Accepts the same options as the CSV datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + years : :class:`~pyspark.sql.Column` or column name, optional + The number of years, positive or negative. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The number of months, positive or negative. + A column that evaluates to an integer. + weeks : :class:`~pyspark.sql.Column` or column name, optional + The number of weeks, positive or negative. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The number of days, positive or negative. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The number of hours, positive or negative. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The number of minutes, positive or negative. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` - A CSV string converted from the given :class:`StructType`. - Returns a column that evaluates to a string. + A new column that contains an interval. + Returns a column that evaluates to an interval. - Examples + See Also -------- - Example 1: Converting a simple StructType to a CSV string - - >>> from pyspark.sql import Row, functions as sf - >>> data = [(1, Row(age=2, name='Alice'))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_csv(df.value)).show() - +-------------+ - |to_csv(value)| - +-------------+ - | 2,Alice| - +-------------+ - - Example 2: Converting a complex StructType to a CSV string - - >>> from pyspark.sql import Row, functions as sf - >>> data = [(1, Row(age=2, name='Alice', scores=[100, 200, 300]))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_csv(df.value)).show(truncate=False) - +-------------------------+ - |to_csv(value) | - +-------------------------+ - |2,Alice,"[100, 200, 300]"| - +-------------------------+ - - Example 3: Converting a StructType with null values to a CSV string - - >>> from pyspark.sql import Row, functions as sf - >>> from pyspark.sql.types import StructType, StructField, IntegerType, StringType - >>> data = [(1, Row(age=None, name='Alice'))] - >>> schema = StructType([ - ... StructField("key", IntegerType(), True), - ... StructField("value", StructType([ - ... StructField("age", IntegerType(), True), - ... StructField("name", StringType(), True) - ... ]), True) - ... ]) - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.to_csv(df.value)).show() - +-------------+ - |to_csv(value)| - +-------------+ - | ,Alice| - +-------------+ - - Example 4: Converting a StructType with different data types to a CSV string - - >>> from pyspark.sql import Row, functions as sf - >>> data = [(1, Row(age=2, name='Alice', isStudent=True))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_csv(df.value)).show() - +-------------+ - |to_csv(value)| - +-------------+ - | 2,Alice,true| - +-------------+ - """ - from pyspark.sql.classic.column import _to_java_column + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.make_dt_interval` + :meth:`pyspark.sql.functions.make_ym_interval` - return _invoke_function("to_csv", _to_java_column(col), _options_to_str(options)) + Examples + -------- + Example 1: Try make interval from years, months, weeks, days, hours, mins and secs. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_interval(df.year, df.month, 'week', df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +---------------------------------------------------------------+ + |try_make_interval(year, month, week, day, hour, min, sec) | + +---------------------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| + +---------------------------------------------------------------+ -@_try_remote_functions -def from_csv( - col: "ColumnOrName", - schema: Union[Column, str], - options: Optional[Mapping[str, str]] = None, -) -> Column: - """ - CSV Function: Parses a column containing a CSV string into a row with the specified schema. - Returns `null` if the string cannot be parsed. + Example 2: Try make interval from years, months, weeks, days, hours and mins. - .. versionadded:: 3.0.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) + ... ).show(truncate=False) + +-------------------------------------------------------+ + |try_make_interval(year, month, week, day, hour, min, 0)| + +-------------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes | + +-------------------------------------------------------+ - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Example 3: Try make interval from years, months, weeks, days and hours. - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - A column or column name in CSV format. - schema : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string, or a DDL-formatted type string, or a DataType. - A column, or Python string literal with schema in DDL format, to use when parsing the CSV column. - options : dict, optional - Options to control parsing. Accepts the same options as the CSV datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_interval(df.year, df.month, 'week', df.day, df.hour) + ... ).show(truncate=False) + +-----------------------------------------------------+ + |try_make_interval(year, month, week, day, hour, 0, 0)| + +-----------------------------------------------------+ + |100 years 11 months 8 days 12 hours | + +-----------------------------------------------------+ - .. # noqa + Example 4: Try make interval from years, months, weeks and days. - Returns - ------- - :class:`~pyspark.sql.Column` - A column of parsed CSV values. - Returns a column that evaluates to a struct. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year, 'month', df.week, df.day)).show(truncate=False) + +--------------------------------------------------+ + |try_make_interval(year, month, week, day, 0, 0, 0)| + +--------------------------------------------------+ + |100 years 11 months 8 days | + +--------------------------------------------------+ - Examples - -------- - Example 1: Parsing a simple CSV string + Example 5: Try make interval from years, months and weeks. - >>> from pyspark.sql import functions as sf - >>> data = [("1,2,3",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, 3}| - +---------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year, 'month', df.week)).show(truncate=False) + +------------------------------------------------+ + |try_make_interval(year, month, week, 0, 0, 0, 0)| + +------------------------------------------------+ + |100 years 11 months 7 days | + +------------------------------------------------+ - Example 2: Using schema_of_csv to infer the schema + Example 6: Try make interval from years and months. - >>> from pyspark.sql import functions as sf - >>> data = [("1,2,3",)] - >>> value = data[0][0] - >>> df.select(sf.from_csv(df.value, sf.schema_of_csv(value))).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, 3}| - +---------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year, 'month')).show(truncate=False) + +---------------------------------------------+ + |try_make_interval(year, month, 0, 0, 0, 0, 0)| + +---------------------------------------------+ + |100 years 11 months | + +---------------------------------------------+ - Example 3: Ignoring leading white space in the CSV string + Example 7: Try make interval from years. - >>> from pyspark.sql import functions as sf - >>> data = [(" abc",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> options = {'ignoreLeadingWhiteSpace': True} - >>> df.select(sf.from_csv(df.value, "s string", options)).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {abc}| - +---------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.try_make_interval(df.year)).show(truncate=False) + +-----------------------------------------+ + |try_make_interval(year, 0, 0, 0, 0, 0, 0)| + +-----------------------------------------+ + |100 years | + +-----------------------------------------+ - Example 4: Parsing a CSV string with a missing value + Example 8: Try make empty interval. - >>> from pyspark.sql import functions as sf - >>> data = [("1,2,",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT")).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, NULL}| - +---------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.try_make_interval()).show(truncate=False) + +--------------------------------------+ + |try_make_interval(0, 0, 0, 0, 0, 0, 0)| + +--------------------------------------+ + |0 seconds | + +--------------------------------------+ - Example 5: Parsing a CSV string with a different delimiter + Example 9: Try make interval from years with overflow. - >>> from pyspark.sql import functions as sf - >>> data = [("1;2;3",)] - >>> df = spark.createDataFrame(data, ("value",)) - >>> options = {'delimiter': ';'} - >>> df.select(sf.from_csv(df.value, "a INT, b INT, c INT", options)).show() - +---------------+ - |from_csv(value)| - +---------------+ - | {1, 2, 3}| - +---------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.try_make_interval(sf.lit(2147483647))).show(truncate=False) + +-----------------------------------------------+ + |try_make_interval(2147483647, 0, 0, 0, 0, 0, 0)| + +-----------------------------------------------+ + |NULL | + +-----------------------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if not isinstance(schema, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "schema", - "arg_type": type(schema).__name__, - }, - ) - - return _invoke_function( - "from_csv", _to_java_column(col), _to_java_column(lit(schema)), _options_to_str(options) + _years = lit(0) if years is None else years + _months = lit(0) if months is None else months + _weeks = lit(0) if weeks is None else weeks + _days = lit(0) if days is None else days + _hours = lit(0) if hours is None else hours + _mins = lit(0) if mins is None else mins + _secs = lit(decimal.Decimal(0)) if secs is None else secs + return _invoke_function_over_columns( + "try_make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs ) -# ---------------------- JSON Functions ---------------------- - - @_try_remote_functions -def get_json_object(col: "ColumnOrName", path: str) -> Column: +def make_interval( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + weeks: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, +) -> Column: """ - Extracts json object from a json string based on json `path` specified, and returns json string - of the extracted json object. It will return null if the input json string is invalid. - - .. versionadded:: 1.6.0 + Make interval from years, months, weeks, days, hours, mins and secs. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - string column in json format. - A column that evaluates to a string. - path : str - path to the json object to extract. - A column that evaluates to a string. + years : :class:`~pyspark.sql.Column` or column name, optional + The number of years, positive or negative. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The number of months, positive or negative. + A column that evaluates to an integer. + weeks : :class:`~pyspark.sql.Column` or column name, optional + The number of weeks, positive or negative. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The number of days, positive or negative. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The number of hours, positive or negative. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The number of minutes, positive or negative. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The number of seconds with the fractional part in microsecond precision. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` - string representation of given JSON object value. - Returns a column that evaluates to a string. + A new column that contains an interval. + Returns a column that evaluates to an interval. - Examples + See Also -------- - Example 1: Extract a json object from json string + :meth:`pyspark.sql.functions.make_dt_interval` + :meth:`pyspark.sql.functions.make_ym_interval` + :meth:`pyspark.sql.functions.try_make_interval` - >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] - >>> df = spark.createDataFrame(data, ("key", "jstring")) - >>> df.select(df.key, - ... get_json_object(df.jstring, '$.f1').alias("c0"), - ... get_json_object(df.jstring, '$.f2').alias("c1") - ... ).show() - +---+-------+------+ - |key| c0| c1| - +---+-------+------+ - | 1| value1|value2| - | 2|value12| NULL| - +---+-------+------+ + Examples + -------- + Example 1: Make interval from years, months, weeks, days, hours, mins and secs. - Example 2: Extract a json object from json array + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +---------------------------------------------------------------+ + |make_interval(year, month, week, day, hour, min, sec) | + +---------------------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes 1.001001 seconds| + +---------------------------------------------------------------+ - >>> data = [ - ... ("1", '''[{"f1": "value1"},{"f1": "value2"}]'''), - ... ("2", '''[{"f1": "value12"},{"f2": "value13"}]''') - ... ] - >>> df = spark.createDataFrame(data, ("key", "jarray")) - >>> df.select(df.key, - ... get_json_object(df.jarray, '$[0].f1').alias("c0"), - ... get_json_object(df.jarray, '$[1].f2').alias("c1") - ... ).show() - +---+-------+-------+ - |key| c0| c1| - +---+-------+-------+ - | 1| value1| NULL| - | 2|value12|value13| - +---+-------+-------+ + Example 2: Make interval from years, months, weeks, days, hours and mins. - >>> df.select(df.key, - ... get_json_object(df.jarray, '$[*].f1').alias("c0"), - ... get_json_object(df.jarray, '$[*].f2').alias("c1") - ... ).show() - +---+-------------------+---------+ - |key| c0| c1| - +---+-------------------+---------+ - | 1|["value1","value2"]| NULL| - | 2| "value12"|"value13"| - +---+-------------------+---------+ - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("get_json_object", _to_java_column(col), _enum_to_value(path)) - - -@_try_remote_functions -def json_tuple(col: "ColumnOrName", *fields: str) -> Column: - """Creates a new row for a json column according to the given field names. - - .. versionadded:: 1.6.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - string column in json format - A column that evaluates to a string. - fields : str - a field or fields to extract - Each a column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - a new row for each given field value from json object - Returns a column that evaluates to a string. - - Examples - -------- - >>> data = [("1", '''{"f1": "value1", "f2": "value2"}'''), ("2", '''{"f1": "value12"}''')] - >>> df = spark.createDataFrame(data, ("key", "jstring")) - >>> df.select(df.key, json_tuple(df.jstring, 'f1', 'f2')).collect() - [Row(key='1', c0='value1', c1='value2'), Row(key='2', c0='value12', c1=None)] - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq - - if len(fields) == 0: - raise PySparkValueError( - errorClass="CANNOT_BE_EMPTY", - messageParameters={"item": "field"}, - ) - sc = _get_active_spark_context() - return _invoke_function("json_tuple", _to_java_column(col), _to_seq(sc, fields)) - - -@_try_remote_functions -def from_json( - col: "ColumnOrName", - schema: Union[ArrayType, StructType, MapType, Column, str], - options: Optional[Mapping[str, str]] = None, -) -> Column: - """ - Parses a column containing a JSON string into a :class:`MapType` with :class:`StringType` - as keys type, :class:`StructType` or :class:`ArrayType` with - the specified schema. Returns `null`, in the case of an unparsable string. - - .. versionadded:: 2.1.0 - - .. versionchanged:: 3.4.0 - Supports Spark Connect. - - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a column or column name in JSON format - schema : :class:`StructType`, :class:`ArrayType`, :class:`MapType`, or str - a StructType, ArrayType of StructType, MapType, or Python string literal with a DDL-formatted string - A column that evaluates to a string, or a DDL-formatted type string, or a DataType. - to use when parsing the json column - options : dict, optional - options to control parsing. accepts the same options as the json datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa - - Returns - ------- - :class:`~pyspark.sql.Column` - a new column of complex type from given JSON object. - Returns a column that evaluates to a struct, array, or map. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour, df.min) + ... ).show(truncate=False) + +---------------------------------------------------+ + |make_interval(year, month, week, day, hour, min, 0)| + +---------------------------------------------------+ + |100 years 11 months 8 days 12 hours 30 minutes | + +---------------------------------------------------+ - Examples - -------- - Example 1: Parsing JSON with a specified schema + Example 3: Make interval from years, months, weeks, days and hours. >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import StructType, StructField, IntegerType - >>> schema = StructType([StructField("a", IntegerType())]) - >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, schema).alias("json")).show() - +----+ - |json| - +----+ - | {1}| - +----+ + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_interval(df.year, df.month, 'week', df.day, df.hour) + ... ).show(truncate=False) + +-------------------------------------------------+ + |make_interval(year, month, week, day, hour, 0, 0)| + +-------------------------------------------------+ + |100 years 11 months 8 days 12 hours | + +-------------------------------------------------+ - Example 2: Parsing JSON with a DDL-formatted string. + Example 4: Make interval from years, months, weeks and days. >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, "a INT").alias("json")).show() - +----+ - |json| - +----+ - | {1}| - +----+ + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year, df.month, 'week', df.day)).show(truncate=False) + +----------------------------------------------+ + |make_interval(year, month, week, day, 0, 0, 0)| + +----------------------------------------------+ + |100 years 11 months 8 days | + +----------------------------------------------+ - Example 3: Parsing JSON into a MapType + Example 5: Make interval from years, months and weeks. >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '''{"a": 1}''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, "MAP").alias("json")).show() - +--------+ - | json| - +--------+ - |{a -> 1}| - +--------+ + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year, df.month, 'week')).show(truncate=False) + +--------------------------------------------+ + |make_interval(year, month, week, 0, 0, 0, 0)| + +--------------------------------------------+ + |100 years 11 months 7 days | + +--------------------------------------------+ - Example 4: Parsing JSON into an ArrayType of StructType + Example 6: Make interval from years and months. >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import ArrayType, StructType, StructField, IntegerType - >>> schema = ArrayType(StructType([StructField("a", IntegerType())])) - >>> df = spark.createDataFrame([(1, '''[{"a": 1}]''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, schema).alias("json")).show() - +-----+ - | json| - +-----+ - |[{1}]| - +-----+ + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year, df.month)).show(truncate=False) + +-----------------------------------------+ + |make_interval(year, month, 0, 0, 0, 0, 0)| + +-----------------------------------------+ + |100 years 11 months | + +-----------------------------------------+ - Example 5: Parsing JSON into an ArrayType + Example 7: Make interval from years. >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import ArrayType, IntegerType - >>> schema = ArrayType(IntegerType()) - >>> df = spark.createDataFrame([(1, '''[1, 2, 3]''')], ("key", "value")) - >>> df.select(sf.from_json(df.value, schema).alias("json")).show() - +---------+ - | json| - +---------+ - |[1, 2, 3]| - +---------+ + >>> df = spark.createDataFrame([[100, 11, 1, 1, 12, 30, 01.001001]], + ... ['year', 'month', 'week', 'day', 'hour', 'min', 'sec']) + >>> df.select(sf.make_interval(df.year)).show(truncate=False) + +-------------------------------------+ + |make_interval(year, 0, 0, 0, 0, 0, 0)| + +-------------------------------------+ + |100 years | + +-------------------------------------+ - Example 6: Parsing JSON with specified options + Example 8: Make empty interval. >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '''{a:123}'''), (2, '''{"a":456}''')], ("key", "value")) - >>> parsed1 = sf.from_json(df.value, "a INT") - >>> parsed2 = sf.from_json(df.value, "a INT", {"allowUnquotedFieldNames": "true"}) - >>> df.select("value", parsed1, parsed2).show() - +---------+----------------+----------------+ - | value|from_json(value)|from_json(value)| - +---------+----------------+----------------+ - | {a:123}| {NULL}| {123}| - |{"a":456}| {456}| {456}| - +---------+----------------+----------------+ + >>> spark.range(1).select(sf.make_interval()).show(truncate=False) + +----------------------------------+ + |make_interval(0, 0, 0, 0, 0, 0, 0)| + +----------------------------------+ + |0 seconds | + +----------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if isinstance(schema, DataType): - schema = schema.json() - elif isinstance(schema, Column): - schema = _to_java_column(schema) - return _invoke_function("from_json", _to_java_column(col), schema, _options_to_str(options)) + _years = lit(0) if years is None else years + _months = lit(0) if months is None else months + _weeks = lit(0) if weeks is None else weeks + _days = lit(0) if days is None else days + _hours = lit(0) if hours is None else hours + _mins = lit(0) if mins is None else mins + _secs = lit(decimal.Decimal(0)) if secs is None else secs + return _invoke_function_over_columns( + "make_interval", _years, _months, _weeks, _days, _hours, _mins, _secs + ) @_try_remote_functions -def to_json(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: +def make_time(hour: "ColumnOrName", minute: "ColumnOrName", second: "ColumnOrName") -> Column: """ - Converts a column containing a :class:`StructType`, :class:`ArrayType`, :class:`MapType` - or a :class:`VariantType` into a JSON string. Throws an exception, in the case of an unsupported type. - - .. versionadded:: 2.1.0 + Create time from hour, minute and second fields. For invalid inputs it will throw an error. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - name of column containing a struct, an array, a map, or a variant object. - A column that evaluates to a struct, array, map, or variant. - options : dict, optional - options to control converting. accepts the same options as the JSON datasource. - See `Data Source Option `_ - for the version you use. - Additionally the function supports the `pretty` option which enables - A dict of options. Each key and value is a string. - pretty JSON generation. - - .. # noqa + hour : :class:`~pyspark.sql.Column` or column name + The hour to represent, from 0 to 23. + A column that evaluates to an integer. + minute : :class:`~pyspark.sql.Column` or column name + The minute to represent, from 0 to 59. + A column that evaluates to an integer. + second : :class:`~pyspark.sql.Column` or column name + The second to represent, from 0 to 59.999999. + A column that evaluates to a decimal. Returns ------- :class:`~pyspark.sql.Column` - JSON object as string column. - Returns a column that evaluates to a string. + A column representing the created time. + Returns a column that evaluates to a time. Examples -------- - Example 1: Converting a StructType column to JSON - - >>> import pyspark.sql.functions as sf - >>> from pyspark.sql import Row - >>> data = [(1, Row(age=2, name='Alice'))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +------------------------+ - |json | - +------------------------+ - |{"age":2,"name":"Alice"}| - +------------------------+ - - Example 2: Converting an ArrayType column to JSON - - >>> import pyspark.sql.functions as sf - >>> from pyspark.sql import Row - >>> data = [(1, [Row(age=2, name='Alice'), Row(age=3, name='Bob')])] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +-------------------------------------------------+ - |json | - +-------------------------------------------------+ - |[{"age":2,"name":"Alice"},{"age":3,"name":"Bob"}]| - +-------------------------------------------------+ - - Example 3: Converting a MapType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, {"name": "Alice"})], ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +----------------+ - |json | - +----------------+ - |{"name":"Alice"}| - +----------------+ - - Example 4: Converting a VariantType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, '{"name": "Alice"}')], ("key", "value")) - >>> df.select(sf.to_json(sf.parse_json(df.value)).alias("json")).show(truncate=False) - +----------------+ - |json | - +----------------+ - |{"name":"Alice"}| - +----------------+ - - Example 5: Converting a nested MapType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, [{"name": "Alice"}, {"name": "Bob"}])], ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +---------------------------------+ - |json | - +---------------------------------+ - |[{"name":"Alice"},{"name":"Bob"}]| - +---------------------------------+ - - Example 6: Converting a simple ArrayType column to JSON - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(1, ["Alice", "Bob"])], ("key", "value")) - >>> df.select(sf.to_json(df.value).alias("json")).show(truncate=False) - +---------------+ - |json | - +---------------+ - |["Alice","Bob"]| - +---------------+ - - Example 7: Converting to JSON with specified options - - >>> import pyspark.sql.functions as sf - >>> df = spark.sql("SELECT (DATE('2022-02-22'), 1) AS date") - >>> json1 = sf.to_json(df.date) - >>> json2 = sf.to_json(df.date, {"dateFormat": "yyyy/MM/dd"}) - >>> df.select("date", json1, json2).show(truncate=False) - +---------------+------------------------------+------------------------------+ - |date |to_json(date) |to_json(date) | - +---------------+------------------------------+------------------------------+ - |{2022-02-22, 1}|{"col1":"2022-02-22","col2":1}|{"col1":"2022/02/22","col2":1}| - +---------------+------------------------------+------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(6, 30, 45.887)], ["hour", "minute", "second"]) + >>> df.select(sf.make_time("hour", "minute", "second").alias("time")).show() + +------------+ + | time| + +------------+ + |06:30:45.887| + +------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("to_json", _to_java_column(col), _options_to_str(options)) + return _invoke_function_over_columns("make_time", hour, minute, second) @_try_remote_functions -def schema_of_json(json: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: +def time_from_seconds(col: "ColumnOrName") -> Column: """ - Parses a JSON string and infers its schema in DDL format. - - .. versionadded:: 2.4.0 + Creates a TIME value from seconds since midnight (supports fractional seconds). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.2.0 Parameters ---------- - json : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a JSON string or a foldable string column containing a JSON string. - options : dict, optional - options to control parsing. accepts the same options as the JSON datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa - - .. versionchanged:: 3.0.0 - It accepts `options` parameter to control schema inferring. - - Returns - ------- - :class:`~pyspark.sql.Column` - a string representation of a :class:`StructType` parsed from given JSON. - Returns a column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + Seconds since midnight (0 to 86399.999999). + A column that evaluates to a numeric. Examples -------- - >>> import pyspark.sql.functions as sf - >>> parsed1 = sf.schema_of_json(sf.lit('{"a": 0}')) - >>> parsed2 = sf.schema_of_json('{a: 1}', {'allowUnquotedFieldNames':'true'}) - >>> spark.range(1).select(parsed1, parsed2).show() - +------------------------+----------------------+ - |schema_of_json({"a": 0})|schema_of_json({a: 1})| - +------------------------+----------------------+ - | STRUCT| STRUCT| - +------------------------+----------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(52200.5,)], ['seconds']) + >>> df.select(sf.time_from_seconds('seconds')).show() + +--------------------------+ + |time_from_seconds(seconds)| + +--------------------------+ + | 14:30:00.5| + +--------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - json = _enum_to_value(json) - if not isinstance(json, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "json", - "arg_type": type(json).__name__, - }, - ) - - return _invoke_function("schema_of_json", _to_java_column(lit(json)), _options_to_str(options)) + return _invoke_function_over_columns("time_from_seconds", col) @_try_remote_functions -def json_array_length(col: "ColumnOrName") -> Column: +def time_from_millis(col: "ColumnOrName") -> Column: """ - Returns the number of elements in the outermost JSON array. `NULL` is returned in case of - any other valid JSON string, `NULL` or an invalid JSON. + Creates a TIME value from milliseconds since midnight. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - col: :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - length of json array. - Returns a column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name + Milliseconds since midnight (0 to 86399999). + A column that evaluates to an integral. Examples -------- - >>> df = spark.createDataFrame([(None,), ('[1, 2, 3]',), ('[]',)], ['data']) - >>> df.select(json_array_length(df.data).alias('r')).collect() - [Row(r=None), Row(r=3), Row(r=0)] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(52200500,)], ['millis']) + >>> df.select(sf.time_from_millis('millis')).show() + +------------------------+ + |time_from_millis(millis)| + +------------------------+ + | 14:30:00.5| + +------------------------+ """ - return _invoke_function_over_columns("json_array_length", col) + return _invoke_function_over_columns("time_from_millis", col) @_try_remote_functions -def json_object_keys(col: "ColumnOrName") -> Column: +def time_from_micros(col: "ColumnOrName") -> Column: """ - Returns all the keys of the outermost JSON object as an array. If a valid JSON object is - given, all the keys of the outermost object will be returned as an array. If it is any - other valid JSON string, an invalid JSON string or an empty string, the function returns null. + Creates a TIME value from microseconds since midnight. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 Parameters ---------- - col: :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - all the keys of the outermost JSON object. - Returns a column that evaluates to an array. + col : :class:`~pyspark.sql.Column` or column name + Microseconds since midnight (0 to 86399999999). + A column that evaluates to an integral. Examples -------- - >>> df = spark.createDataFrame([(None,), ('{}',), ('{"key1":1, "key2":2}',)], ['data']) - >>> df.select(json_object_keys(df.data).alias('r')).collect() - [Row(r=None), Row(r=[]), Row(r=['key1', 'key2'])] + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(52200500000,)], ['micros']) + >>> df.select(sf.time_from_micros('micros')).show() + +------------------------+ + |time_from_micros(micros)| + +------------------------+ + | 14:30:00.5| + +------------------------+ """ - return _invoke_function_over_columns("json_object_keys", col) + return _invoke_function_over_columns("time_from_micros", col) @_try_remote_functions -def json_typeof(col: "ColumnOrName") -> Column: +def time_to_seconds(col: "ColumnOrName") -> Column: """ - Returns the type of the outermost JSON value as a string: one of 'object', 'array', - 'string', 'number', 'boolean', or 'null'. Returns null if the input is not a valid JSON - string or is an empty string. + Extracts seconds from TIME value (returns DECIMAL to preserve fractional seconds). - .. versionadded:: 4.4.0 + .. versionadded:: 4.2.0 Parameters ---------- - col: :class:`~pyspark.sql.Column` or str - target column to compute on. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - the type of the outermost JSON value. - Returns a column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + TIME value to convert. - See Also + Examples -------- - :meth:`pyspark.sql.functions.json_object_keys` - :meth:`pyspark.sql.functions.get_json_object` - :meth:`pyspark.sql.functions.json_array_length` - - Examples - -------- - >>> df = spark.createDataFrame([('{"a": 1}',), ('[1, 2, 3]',), ('123',), ('',)], ['data']) - >>> df.select(json_typeof(df.data).alias('r')).collect() - [Row(r='object'), Row(r='array'), Row(r='number'), Row(r=None)] + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") + >>> df.select(sf.time_to_seconds('time')).show() + +---------------------+ + |time_to_seconds(time)| + +---------------------+ + | 52200.500000| + +---------------------+ """ - return _invoke_function_over_columns("json_typeof", col) - - -# ---------------------- VARIANT Functions ---------------------- + return _invoke_function_over_columns("time_to_seconds", col) @_try_remote_functions -def try_parse_json( - col: "ColumnOrName", -) -> Column: +def time_to_millis(col: "ColumnOrName") -> Column: """ - Parses a column containing a JSON string into a :class:`VariantType`. Returns None if a string - contains an invalid JSON value. + Extracts milliseconds from TIME value. - .. versionadded:: 4.0.0 + .. versionadded:: 4.2.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings. - A column that evaluates to a string. - - Returns - ------- - :class:`~pyspark.sql.Column` - a new column of VariantType. - Returns a column that evaluates to a variant. + col : :class:`~pyspark.sql.Column` or column name + TIME value to convert. Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''}, {'json': '''{a : 1}'''} ]) - >>> df.select(to_json(try_parse_json(df.json))).collect() - [Row(to_json(try_parse_json(json))='{"a":1}'), Row(to_json(try_parse_json(json))=None)] + >>> from pyspark.sql import functions as sf + >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") + >>> df.select(sf.time_to_millis('time')).show() + +--------------------+ + |time_to_millis(time)| + +--------------------+ + | 52200500| + +--------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("try_parse_json", _to_java_column(col)) + return _invoke_function_over_columns("time_to_millis", col) @_try_remote_functions -def to_variant_object( - col: "ColumnOrName", -) -> Column: +def time_to_micros(col: "ColumnOrName") -> Column: """ - Converts a column containing nested inputs (array/map/struct) into a variants where maps and - structs are converted to variant objects which are unordered unlike SQL structs. Input maps can - only have string keys. + Extracts microseconds from TIME value. - .. versionadded:: 4.0.0 + .. versionadded:: 4.2.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - a column with a nested schema or column name - A column that evaluates to an array, map, or struct. - - Returns - ------- - :class:`~pyspark.sql.Column` - a new column of VariantType. - Returns a column that evaluates to a variant. + col : :class:`~pyspark.sql.Column` or column name + TIME value to convert. Examples -------- - Example 1: Converting an array containing a nested struct into a variant - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, StructType, StructField, StringType, MapType - >>> schema = StructType([ - ... StructField("i", StringType(), True), - ... StructField("v", ArrayType(StructType([ - ... StructField("a", MapType(StringType(), StringType()), True) - ... ]), True)) - ... ]) - >>> data = [("1", [{"a": {"b": 2}}])] - >>> df = spark.createDataFrame(data, schema) - >>> df.select(sf.to_variant_object(df.v)) - DataFrame[to_variant_object(v): variant] - >>> df.select(sf.to_variant_object(df.v)).show(truncate=False) + >>> df = spark.sql("SELECT TIME'14:30:00.5' as time") + >>> df.select(sf.time_to_micros('time')).show() +--------------------+ - |to_variant_object(v)| + |time_to_micros(time)| +--------------------+ - |[{"a":{"b":"2"}}] | + | 52200500000| +--------------------+ """ - from pyspark.sql.classic.column import _to_java_column + return _invoke_function_over_columns("time_to_micros", col) - return _invoke_function("to_variant_object", _to_java_column(col)) +def _ensure_column_or_name(arg: Optional[Any]) -> "ColumnOrName": + if not isinstance(arg, (Column, str)): + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "Column or str", + "arg_name": "arg", + "arg_type": type(arg).__name__, + }, + ) + return arg -@_try_remote_functions -def variant_from_arrays(keys: "ColumnOrName", values: "ColumnOrName") -> Column: - """ - Creates a variant object from the given arrays of keys and values. The keys must be non-null - strings and the two arrays must have the same length. - .. versionadded:: 4.4.0 +@overload +def make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... - Parameters - ---------- - keys : :class:`~pyspark.sql.Column` or column name - an array of string keys. - values : :class:`~pyspark.sql.Column` or column name - an array of values. - Returns - ------- - :class:`~pyspark.sql.Column` - a new column of VariantType. +@overload +def make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: "ColumnOrName", +) -> Column: ... - See Also - -------- - :meth:`pyspark.sql.functions.variant_from_entries` - :meth:`pyspark.sql.functions.to_variant_object` - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array('a', 'b') AS keys, array(1, 2) AS values") - >>> df.select(sf.variant_from_arrays("keys", "values").cast("string").alias("r")).collect() - [Row(r='{"a":1,"b":2}')] - """ - return _invoke_function_over_columns("variant_from_arrays", keys, values) +@overload +def make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... + + +@overload +def make_timestamp( + *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" +) -> Column: ... @_try_remote_functions -def variant_from_entries(entries: "ColumnOrName") -> Column: +def make_timestamp( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + timezone: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Creates a variant object from an array of key/value struct entries. The keys must be non-null - strings. + Create timestamp from years, months, days, hours, mins, secs, and (optional) timezone fields. + Alternatively, create timestamp from date, time, and (optional) timezone fields. + The result data type is consistent with the value of configuration `spark.sql.timestampType`. + If the configuration `spark.sql.ansi.enabled` is false, the function returns NULL + on invalid inputs. Otherwise, it will throw an error instead. - .. versionadded:: 4.4.0 + .. versionadded:: 3.5.0 + + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. Parameters ---------- - entries : :class:`~pyspark.sql.Column` or column name - an array of key/value structs, where the first field is a string key and the second field - is the value. + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or column name, optional + The time zone identifier. For example, CET, UTC, and etc. + A column that evaluates to a string. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - a new column of VariantType. + A new column that contains a timestamp. + Returns a column that evaluates to a timestamp. See Also -------- - :meth:`pyspark.sql.functions.variant_from_arrays` - :meth:`pyspark.sql.functions.to_variant_object` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.sql("SELECT array(struct('a', 1), struct('b', 2)) AS entries") - >>> df.select(sf.variant_from_entries("entries").cast("string").alias("r")).collect() - [Row(r='{"a":1,"b":2}')] - """ - return _invoke_function_over_columns("variant_from_entries", entries) + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. -@_try_remote_functions -def parse_json( - col: "ColumnOrName", -) -> Column: - """ - Parses a column containing a JSON string into a :class:`VariantType`. Throws exception if a - string represents an invalid JSON value. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') + ... ).show(truncate=False) + +----------------------------------------------------+ + |make_timestamp(year, month, day, hour, min, sec, tz)| + +----------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +----------------------------------------------------+ - .. versionadded:: 4.0.0 + Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). - Parameters - ---------- - col : :class:`~pyspark.sql.Column` or str - a column or column name JSON formatted strings. - A column that evaluates to a string. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +------------------------------------------------+ + |make_timestamp(year, month, day, hour, min, sec)| + +------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - a new column of VariantType. - Returns a column that evaluates to a variant. + Example 3: Make timestamp from date, time, and timezone. - Examples - -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(to_json(parse_json(df.json))).collect() - [Row(to_json(parse_json(json))='{"a":1}')] - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time"), + ... sf.lit("CET").alias("tz") + ... ) + >>> df.select( + ... sf.make_timestamp(date=df.date, time=df.time, timezone=df.tz) + ... ).show(truncate=False) + +------------------------------+ + |make_timestamp(date, time, tz)| + +------------------------------+ + |2014-12-27 21:30:45.887 | + +------------------------------+ - return _invoke_function("parse_json", _to_java_column(col)) + Example 4: Make timestamp from date and time (without timezone). + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time") + ... ) + >>> df.select(sf.make_timestamp(date=df.date, time=df.time)).show(truncate=False) + +--------------------------+ + |make_timestamp(date, time)| + +--------------------------+ + |2014-12-28 06:30:45.887 | + +--------------------------+ -@_try_remote_functions -def is_variant_null(v: "ColumnOrName") -> Column: + >>> spark.conf.unset("spark.sql.session.timeZone") """ - Check if a variant value is a variant null. Returns true if and only if the input is a variant - null and false otherwise (including in the case of SQL NULL). + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) + else: + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "make_timestamp", + _ensure_column_or_name(date), + _ensure_column_or_name(time), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) - .. versionadded:: 4.0.0 - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. +@overload +def try_make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... - Returns - ------- - :class:`~pyspark.sql.Column` - a boolean column indicating whether the variant value is a variant null - Returns a column that evaluates to a boolean. - Examples - -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(is_variant_null(parse_json(df.json)).alias("r")).collect() - [Row(r=False)] - """ - from pyspark.sql.classic.column import _to_java_column +@overload +def try_make_timestamp( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: "ColumnOrName", +) -> Column: ... - return _invoke_function("is_variant_null", _to_java_column(v)) + +@overload +def try_make_timestamp(*, date: "ColumnOrName", time: "ColumnOrName") -> Column: ... + + +@overload +def try_make_timestamp( + *, date: "ColumnOrName", time: "ColumnOrName", timezone: "ColumnOrName" +) -> Column: ... @_try_remote_functions -def is_valid_variant(v: "ColumnOrName") -> Column: +def try_make_timestamp( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + timezone: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Check if a variant value is valid. Returns true if the variant is valid, false if it is - malformed, and NULL if the input is NULL. + Try to create timestamp from years, months, days, hours, mins, secs and (optional) timezone + fields. Alternatively, try to create timestamp from date, time, and (optional) timezone fields. + The result data type is consistent with the value of configuration `spark.sql.timestampType`. + The function returns NULL on invalid inputs. - .. versionadded:: 4.2.0 + .. versionadded:: 4.0.0 + + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or column name, optional + The time zone identifier. For example, CET, UTC, and etc. + A column that evaluates to a string. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - a boolean column indicating whether the variant value is valid - Returns a column that evaluates to a boolean. + A new column that contains a timestamp or NULL in case of an error. + Returns a column that evaluates to a timestamp. - Examples + See Also -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(is_valid_variant(parse_json(df.json)).alias("r")).collect() - [Row(r=True)] - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("is_valid_variant", _to_java_column(v)) - - -@_try_remote_functions -def variant_delete(v: "ColumnOrName", *paths: Union[Column, str]) -> Column: - """ - Removes fields or array elements from a variant at the given JSONPath locations. - Multiple paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are - skipped. - - .. versionadded:: 5.0.0 - - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - paths : :class:`~pyspark.sql.Column` or str - one or more JSONPath deletion targets. A `str` is a literal path; a - :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path - should start with `$` and is followed by one or more segments like - `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not - A column that evaluates to a string. - allowed. - - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with the specified paths removed - Returns a column that evaluates to a variant. - - Examples - -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_delete - >>> df = spark.createDataFrame([{ - ... 'json': '''{ "a" : 1, "b" : 2, "c" : 3, "items" : [1, 2, 3] }''', - ... 'path': '$.a' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_delete(v, lit(None), "$.a", "$.c")).alias("r")).collect() - [Row(r='{"b":2,"items":[1,2,3]}')] - >>> df.select(to_json(variant_delete(v, "$.missing")).alias("r")).collect() - [Row(r='{"a":1,"b":2,"c":3,"items":[1,2,3]}')] - >>> df.select(to_json(variant_delete(v, df.path)).alias("r")).collect() - [Row(r='{"b":2,"c":3,"items":[1,2,3]}')] - >>> df.select(to_json(variant_delete(v, "$.items[0]", "$.items[0]")).alias("r")).collect() - [Row(r='{"a":1,"b":2,"c":3,"items":[3]}')] - >>> df.select(variant_delete(lit(None), "$.a").alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column, _to_seq - - if len(paths) == 0: - raise PySparkValueError( - errorClass="CANNOT_BE_EMPTY", - messageParameters={"item": "paths"}, - ) - sc = _get_active_spark_context() - - path_cols = [p if isinstance(p, Column) else lit(p) for p in paths] - return _invoke_function( - "variant_delete", - _to_java_column(v), - _to_java_column(path_cols[0]), - _to_seq(sc, path_cols[1:], _to_java_column), - ) - - -@_try_remote_functions -def variant_insert(v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName") -> Column: - """ - Inserts a value into a variant at the given JSONPath location. An object path adds a new field - (error if it already exists); an array path inserts at the index, shifting later elements - right. Missing intermediate keys are created. Throws an error if a path segment hits a value - of an incompatible type. Returns NULL if any argument is NULL. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath insertion target. A `str` is a literal path; a - :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with - `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or - A column that evaluates to a string. - `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to insert. Any expression castable to variant. - - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with `value` inserted at `path` - Returns a column that evaluates to a variant. - - Examples - -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_insert - >>> df = spark.createDataFrame([{ - ... 'json': '''{ "a": 1, "arr": ["x", "y"] }''', - ... 'path': '$.d' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_insert(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"b":2}')] - >>> df.select(to_json(variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] - >>> df.select(to_json(variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","z","y"]}')] - >>> df.select(to_json(variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] - >>> df.select(to_json(variant_insert(v, df.path, lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"d":9}')] - >>> df.select(to_json(variant_insert(v, "$.b", parse_json(lit('null')))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"b":null}')] - >>> df.select(variant_insert(v, "$.b", lit(None)).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column - - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "variant_insert", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) - - -@_try_remote_functions -def try_variant_insert( - v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" -) -> Column: - """ - Inserts a value into a variant at the given JSONPath location. An object path adds a new field; - an array path inserts at the index, shifting later elements right. Missing intermediate keys - are created. Returns NULL if the field already exists or a path segment hits a value of an - incompatible type, or if any argument is NULL. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath insertion target. A `str` is a literal path; a - :class:`~pyspark.sql.Column` supplies the path at runtime. A valid path should start with - `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, or - `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to insert. Any expression castable to variant. - - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with `value` inserted at `path`, or NULL if the insertion fails - Returns a column that evaluates to a variant. + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_insert - >>> df = spark.createDataFrame([{'json': '''{ "a": 1, "arr": ["x", "y"] }'''}]) - >>> v = parse_json(df.json) - >>> df.select(to_json(try_variant_insert(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"b":2}')] - >>> df.select(to_json(try_variant_insert(v, "$.c.d", lit(3))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y"],"c":{"d":3}}')] - >>> df.select(to_json(try_variant_insert(v, "$.arr[1]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","z","y"]}')] - >>> df.select(to_json(try_variant_insert(v, "$.arr[5]", lit("z"))).alias("r")).collect() - [Row(r='{"a":1,"arr":["x","y",null,null,null,"z"]}')] - >>> df.select(to_json(try_variant_insert(v, "$.a", lit(2))).alias("r")).collect() - [Row(r=None)] - >>> df.select(to_json(try_variant_insert(v, "$.a.b", lit(2))).alias("r")).collect() - [Row(r=None)] - >>> df.select(to_json(try_variant_insert(v, "$.b", lit(None))).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column - - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "try_variant_insert", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) - - -@_try_remote_functions -def variant_set( - v: "ColumnOrName", - path: Union[Column, str], - value: "ColumnOrName", - create_if_missing: bool = True, -) -> Column: - """ - Sets or upserts a value in a variant at the given JSONPath location. An existing object field - or array element at the target is replaced. A missing field, array index, or intermediate path - is created, unless `create_if_missing` is false, in which case the variant is left unchanged. - Throws an error if a path segment hits a value of an incompatible type. Returns NULL if any - argument is NULL. + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - .. versionadded:: 4.3.0 + Example 1: Make timestamp from years, months, days, hours, mins, secs, and timezone. - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies - the path at runtime. A valid path should start with `$` and is followed by one or more - A column that evaluates to a string. - segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to set. Any expression castable to variant. - create_if_missing : bool, optional - whether to create missing keys or out-of-range array indices (default True). - A column that evaluates to a boolean. Must be a constant. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec, 'tz') + ... ).show(truncate=False) + +----------------------------------------------------+ + |try_make_timestamp(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +----------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with `value` set at `path` - Returns a column that evaluates to a variant. + Example 2: Make timestamp from years, months, days, hours, mins, and secs (without timezone). - Examples - -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_set - >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_set(v, "$.a", lit(9))).alias("r")).collect() - [Row(r='{"a":9,"arr":[1,2,3]}')] - >>> df.select(to_json(variant_set(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] - >>> df.select(to_json(variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,9,3]}')] - >>> df.select(to_json(variant_set(v, "$.b", lit(2), False)).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3]}')] - >>> df.select(to_json(variant_set(v, "$.a", parse_json(lit("null")))).alias("r")).collect() - [Row(r='{"a":null,"arr":[1,2,3]}')] - >>> df.select(to_json(variant_set(v, "$.a", lit(None))).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |try_make_timestamp(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +----------------------------------------------------+ - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "variant_set", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - _enum_to_value(create_if_missing), - ) + Example 3: Make timestamp with invalid input. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp(df.year, df.month, df.day, 'hour', df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |try_make_timestamp(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |NULL | + +----------------------------------------------------+ -@_try_remote_functions -def try_variant_set( - v: "ColumnOrName", - path: Union[Column, str], - value: "ColumnOrName", - create_if_missing: bool = True, -) -> Column: - """ - Sets or upserts a value in a variant at the given JSONPath location. An existing object field - or array element at the target is replaced. A missing field, array index, or intermediate path - is created, unless `create_if_missing` is false, in which case the variant is left unchanged. - Returns NULL if a path segment hits a value of an incompatible type, or if any argument is NULL. + Example 4: Make timestamp from date, time, and timezone. - .. versionadded:: 4.3.0 + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time"), + ... sf.lit("CET").alias("tz") + ... ) + >>> df.select( + ... sf.try_make_timestamp(date=df.date, time=df.time, timezone=df.tz) + ... ).show(truncate=False) + +----------------------------------+ + |try_make_timestamp(date, time, tz)| + +----------------------------------+ + |2014-12-27 21:30:45.887 | + +----------------------------------+ - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - path : :class:`~pyspark.sql.Column` or str - the JSONPath set target. A `str` is a literal path; a :class:`~pyspark.sql.Column` supplies - the path at runtime. A valid path should start with `$` and is followed by one or more - segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` is not allowed. - value : :class:`~pyspark.sql.Column` or str - the value to set. Any expression castable to variant. - create_if_missing : bool, optional - whether to create missing keys or out-of-range array indices (default True). + Example 5: Make timestamp from date and time (without timezone). - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with `value` set at `path` + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time") + ... ) + >>> df.select(sf.try_make_timestamp(date=df.date, time=df.time)).show(truncate=False) + +------------------------------+ + |try_make_timestamp(date, time)| + +------------------------------+ + |2014-12-28 06:30:45.887 | + +------------------------------+ - Examples - -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_set - >>> df = spark.createDataFrame([{'json': '''{"a": 1, "arr": [1, 2, 3]}'''}]) - >>> v = parse_json(df.json) - >>> df.select(to_json(try_variant_set(v, "$.a", lit(9))).alias("r")).collect() - [Row(r='{"a":9,"arr":[1,2,3]}')] - >>> df.select(to_json(try_variant_set(v, "$.b", lit(2))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3],"b":2}')] - >>> df.select(to_json(try_variant_set(v, "$.arr[1]", lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,9,3]}')] - >>> df.select(to_json(try_variant_set(v, "$.arr[5]", lit(9))).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3,null,null,9]}')] - >>> df.select(to_json(try_variant_set(v, "$.b", lit(2), False)).alias("r")).collect() - [Row(r='{"a":1,"arr":[1,2,3]}')] - >>> df.select(to_json(try_variant_set(v, "$.a.b", lit(9))).alias("r")).collect() - [Row(r=None)] - >>> df.select(to_json(try_variant_set(v, "$.a", lit(None))).alias("r")).collect() - [Row(r=None)] + >>> spark.conf.unset("spark.sql.session.timeZone") """ - from pyspark.sql.classic.column import _to_java_column - - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "try_variant_set", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - _enum_to_value(create_if_missing), - ) + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "try_make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "try_make_timestamp", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) + else: + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + if timezone is not None: + return _invoke_function_over_columns( + "try_make_timestamp", + _ensure_column_or_name(date), + _ensure_column_or_name(time), + _ensure_column_or_name(timezone), + ) + else: + return _invoke_function_over_columns( + "try_make_timestamp", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) @_try_remote_functions -def variant_array_append( - v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" +def make_timestamp_ltz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: Optional["ColumnOrName"] = None, ) -> Column: """ - Appends a value to the array in a variant at the given JSONPath location. Returns the variant - unchanged if a path key or index is absent. Throws an error if a path segment hits a value of - an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + Create the current timestamp with local time zone from years, months, days, hours, mins, + secs and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, + the function returns NULL on invalid inputs. Otherwise, it will throw an error instead. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` - supplies the path at runtime. A valid path should start with `$` and is followed by zero or + years : :class:`~pyspark.sql.Column` or str + The year to represent, from 1 to 9999. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or str + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or str + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or str + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or str + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or str + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13 , or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or str, optional + The time zone identifier. For example, CET, UTC and etc. A column that evaluates to a string. - more segments like `[123]`, `.name`, `['name']`, or `["name"]`. - value : :class:`~pyspark.sql.Column` or str - the value to append. Any expression castable to variant. - - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with `value` appended to the array at `path` - Returns a column that evaluates to a variant. - - Examples - -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_array_append - >>> df = spark.createDataFrame([{ - ... 'json': '''[[1, 2], 5]''', - ... 'path': '$[0]' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_array_append(v, "$", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5,3]')] - >>> df.select(to_json(variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5]')] - >>> df.select(to_json(variant_array_append(v, df.path, lit(9))).alias("r")).collect() - [Row(r='[[1,2,9],5]')] - >>> nested = variant_array_append(v, "$", parse_json(lit('[4, 5]'))) - >>> df.select(to_json(nested).alias("r")).collect() - [Row(r='[[1,2],5,[4,5]]')] - >>> df.select(variant_array_append(v, "$", lit(None)).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column - - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "variant_array_append", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) - - -@_try_remote_functions -def try_variant_array_append( - v: "ColumnOrName", path: Union[Column, str], value: "ColumnOrName" -) -> Column: - """ - Appends a value to the array in a variant at the given JSONPath location. Returns the variant - unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an - incompatible type, the target is not an array, or if any argument is NULL. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - path : :class:`~pyspark.sql.Column` or str - the JSONPath target array. A `str` is a literal path; a :class:`~pyspark.sql.Column` - supplies the path at runtime. A valid path should start with `$` and is followed by zero or - more segments like `[123]`, `.name`, `['name']`, or `["name"]`. - value : :class:`~pyspark.sql.Column` or str - the value to append. Any expression castable to variant. Returns ------- :class:`~pyspark.sql.Column` - a variant column with `value` appended to the array at `path` + A new column that contains a current timestamp. + Returns a column that evaluates to a timestamp. - Examples + See Also -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, try_variant_array_append - >>> df = spark.createDataFrame([{ - ... 'json': '''[[1, 2], 5]''', - ... 'path': '$[0]' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(try_variant_array_append(v, "$", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5,3]')] - >>> df.select(to_json(try_variant_array_append(v, "$[5]", lit(3))).alias("r")).collect() - [Row(r='[[1,2],5]')] - >>> df.select(to_json(try_variant_array_append(v, df.path, lit(9))).alias("r")).collect() - [Row(r='[[1,2,9],5]')] - >>> df.select(to_json(try_variant_array_append(v, "$[1]", lit(9))).alias("r")).collect() - [Row(r=None)] - >>> df.select(try_variant_array_append(v, "$", lit(None)).alias("r")).collect() - [Row(r=None)] - """ - from pyspark.sql.classic.column import _to_java_column - - path_col = path if isinstance(path, Column) else lit(path) - return _invoke_function( - "try_variant_array_append", - _to_java_column(v), - _to_java_column(path_col), - _to_java_column(value), - ) - - -@_try_remote_functions -def variant_strip_nulls(v: "ColumnOrName", include_arrays: bool = True) -> Column: - """ - Recursively removes object fields and array elements whose value is a variant null, unless - `include_arrays` is False, in which case null array elements are kept. Returns NULL if any - argument is NULL. - - .. versionadded:: 4.3.0 - - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - include_arrays : bool, optional - whether null elements are also removed from arrays (default True). - - Returns - ------- - :class:`~pyspark.sql.Column` - a variant column with variant null fields/elements removed + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql.functions import lit, parse_json, to_json, variant_strip_nulls - >>> df = spark.createDataFrame([{ - ... 'json': '''{ "a" : 1, "b" : null, "c" : [1, null], "d" : { "e" : null, "f" : 4 } }''' - ... }]) - >>> v = parse_json(df.json) - >>> df.select(to_json(variant_strip_nulls(v)).alias("r")).collect() - [Row(r='{"a":1,"c":[1],"d":{"f":4}}')] - >>> df.select(to_json(variant_strip_nulls(v, False)).alias("r")).collect() - [Row(r='{"a":1,"c":[1,null],"d":{"f":4}}')] - >>> df.select(variant_strip_nulls(lit(None)).alias("r")).collect() - [Row(r=None)] - >>> df2 = spark.createDataFrame([{'json': '{"a": null}'}, {'json': 'null'}]) - >>> v2 = parse_json(df2.json) - >>> df2.select(to_json(variant_strip_nulls(v2)).alias("r")).collect() - [Row(r='{}'), Row(r='null')] - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function( - "variant_strip_nulls", _to_java_column(v), _enum_to_value(include_arrays) - ) - - -@_try_remote_functions -def variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: - """ - Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to - `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - .. versionadded:: 4.0.0 + Example 1: Make the current timestamp from years, months, days, hours, mins and secs. - Parameters - ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - a column containing the extraction path strings or a string representing the extraction - path. A valid path should start with `$` and is followed by zero or more segments like - A column that evaluates to a string. - `[123]`, `.name`, `['name']`, or `["name"]`. - targetType : str - A DDL-formatted type string. Must be a constant. - the target data type to cast into, in a DDL-formatted string + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec, 'tz') + ... ).show(truncate=False) + +--------------------------------------------------------+ + |make_timestamp_ltz(year, month, day, hour, min, sec, tz)| + +--------------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +--------------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - a column of `targetType` representing the extracted result - Returns a column of the type given by `targetType`. + Example 2: Make the current timestamp without timezone. - Examples - -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) - >>> df.select(variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() - [Row(r=1)] - >>> df.select(variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() - [Row(r=None)] - >>> df.select(variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() - [Row(r=1)] - """ - from pyspark.sql.classic.column import _to_java_column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.make_timestamp_ltz(df.year, df.month, 'day', df.hour, df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |make_timestamp_ltz(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +----------------------------------------------------+ - assert isinstance(path, (Column, str)) - if isinstance(path, str): - return _invoke_function( - "variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) + >>> spark.conf.unset("spark.sql.session.timeZone") + """ + if timezone is not None: + return _invoke_function_over_columns( + "make_timestamp_ltz", years, months, days, hours, mins, secs, timezone ) else: - return _invoke_function( - "variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) + return _invoke_function_over_columns( + "make_timestamp_ltz", years, months, days, hours, mins, secs ) @_try_remote_functions -def try_variant_get(v: "ColumnOrName", path: Union[Column, str], targetType: str) -> Column: +def try_make_timestamp_ltz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", + timezone: Optional["ColumnOrName"] = None, +) -> Column: """ - Extracts a sub-variant from `v` according to `path`, and then cast the sub-variant to - `targetType`. Returns null if the path does not exist or the cast fails. + Try to create the current timestamp with local time zone from years, months, days, hours, mins, + secs and timezone fields. + The function returns NULL on invalid inputs. .. versionadded:: 4.0.0 Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. - path : :class:`~pyspark.sql.Column` or str - a column containing the extraction path strings or a string representing the extraction - path. A valid path should start with `$` and is followed by zero or more segments like + years : :class:`~pyspark.sql.Column` or column name + The year to represent, from 1 to 9999. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name + The month-of-year to represent, from 1 (January) to 12 (December). + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name + The day-of-month to represent, from 1 to 31. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name + The hour-of-day to represent, from 0 to 23. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name + The minute-of-hour to represent, from 0 to 59. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13 , or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + A column that evaluates to a decimal. + timezone : :class:`~pyspark.sql.Column` or column name, optional + The time zone identifier. For example, CET, UTC and etc. A column that evaluates to a string. - `[123]`, `.name`, `['name']`, or `["name"]`. - targetType : str - A DDL-formatted type string. Must be a constant. - the target data type to cast into, in a DDL-formatted string Returns ------- :class:`~pyspark.sql.Column` - a column of `targetType` representing the extracted result - Returns a column of the type given by `targetType`. + A new column that contains a current timestamp, or NULL in case of an error. + Returns a column that evaluates to a timestamp. + + See Also + -------- + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }''', 'path': '$.a'} ]) - >>> df.select(try_variant_get(parse_json(df.json), "$.a", "int").alias("r")).collect() - [Row(r=1)] - >>> df.select(try_variant_get(parse_json(df.json), "$.b", "int").alias("r")).collect() - [Row(r=None)] - >>> df.select(try_variant_get(parse_json(df.json), "$.a", "binary").alias("r")).collect() - [Row(r=None)] - >>> df.select(try_variant_get(parse_json(df.json), df.path, "int").alias("r")).collect() - [Row(r=1)] - """ - from pyspark.sql.classic.column import _to_java_column + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - if isinstance(path, str): - return _invoke_function( - "try_variant_get", _to_java_column(v), _enum_to_value(path), _enum_to_value(targetType) + Example 1: Make the current timestamp from years, months, days, hours, mins and secs. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec, 'tz') + ... ).show(truncate=False) + +------------------------------------------------------------+ + |try_make_timestamp_ltz(year, month, day, hour, min, sec, tz)| + +------------------------------------------------------------+ + |2014-12-27 21:30:45.887 | + +------------------------------------------------------------+ + + Example 2: Make the current timestamp without timezone. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ltz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +--------------------------------------------------------+ + + Example 3: Make the current timestamp with invalid input. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887, 'CET']], + ... ['year', 'month', 'day', 'hour', 'min', 'sec', 'tz']) + >>> df.select( + ... sf.try_make_timestamp_ltz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ltz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |NULL | + +--------------------------------------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") + """ + if timezone is not None: + return _invoke_function_over_columns( + "try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone ) else: - return _invoke_function( - "try_variant_get", _to_java_column(v), _to_java_column(path), _enum_to_value(targetType) + return _invoke_function_over_columns( + "try_make_timestamp_ltz", years, months, days, hours, mins, secs ) +@overload +def make_timestamp_ntz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... + + +@overload +def make_timestamp_ntz( + *, + date: "ColumnOrName", + time: "ColumnOrName", +) -> Column: ... + + @_try_remote_functions -def schema_of_variant(v: "ColumnOrName") -> Column: +def make_timestamp_ntz( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Returns schema in the SQL format of a variant. + Create local date-time from years, months, days, hours, mins, secs fields. Alternatively, try to + create local date-time from date and time fields. If the configuration `spark.sql.ansi.enabled` + is false, the function returns NULL on invalid inputs. Otherwise, it will throw an error. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 + + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - a string column representing the variant schema - Returns a column that evaluates to a string. + A new column that contains a local date-time. + Returns a column that evaluates to a timestamp. + + See Also + -------- + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.try_make_timestamp_ntz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(schema_of_variant(parse_json(df.json)).alias("r")).collect() - [Row(r='OBJECT')] + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + + Example 1: Make local date-time from years, months, days, hours, mins, secs. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +----------------------------------------------------+ + |make_timestamp_ntz(year, month, day, hour, min, sec)| + +----------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +----------------------------------------------------+ + + Example 2: Make local date-time from date and time. + + >>> import pyspark.sql.functions as sf + >>> from datetime import date, time + >>> df = spark.range(1).select( + ... sf.lit(date(2014, 12, 28)).alias("date"), + ... sf.lit(time(6, 30, 45, 887000)).alias("time") + ... ) + >>> df.select(sf.make_timestamp_ntz(date=df.date, time=df.time)).show(truncate=False) + +------------------------------+ + |make_timestamp_ntz(date, time)| + +------------------------------+ + |2014-12-28 06:30:45.887 | + +------------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - from pyspark.sql.classic.column import _to_java_column + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "make_timestamp_ntz", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) + else: + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) - return _invoke_function("schema_of_variant", _to_java_column(v)) + +@overload +def try_make_timestamp_ntz( + years: "ColumnOrName", + months: "ColumnOrName", + days: "ColumnOrName", + hours: "ColumnOrName", + mins: "ColumnOrName", + secs: "ColumnOrName", +) -> Column: ... + + +@overload +def try_make_timestamp_ntz( + *, + date: "ColumnOrName", + time: "ColumnOrName", +) -> Column: ... @_try_remote_functions -def schema_of_variant_agg(v: "ColumnOrName") -> Column: +def try_make_timestamp_ntz( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, + days: Optional["ColumnOrName"] = None, + hours: Optional["ColumnOrName"] = None, + mins: Optional["ColumnOrName"] = None, + secs: Optional["ColumnOrName"] = None, + date: Optional["ColumnOrName"] = None, + time: Optional["ColumnOrName"] = None, +) -> Column: """ - Returns the merged schema in the SQL format of a variant column. + Try to create local date-time from years, months, days, hours, mins, secs fields. Alternatively, + try to create local date-time from date and time fields. The function returns NULL on invalid + inputs. .. versionadded:: 4.0.0 + .. versionchanged:: 4.1.0 + Added support for creating timestamps from date and time. + Parameters ---------- - v : :class:`~pyspark.sql.Column` or str - a variant column or column name - A column that evaluates to a variant. + years : :class:`~pyspark.sql.Column` or column name, optional + The year to represent, from 1 to 9999. + Required when creating timestamps from individual components. + Must be used with months, days, hours, mins, and secs. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The month-of-year to represent, from 1 (January) to 12 (December). + Required when creating timestamps from individual components. + Must be used with years, days, hours, mins, and secs. + A column that evaluates to an integer. + days : :class:`~pyspark.sql.Column` or column name, optional + The day-of-month to represent, from 1 to 31. + Required when creating timestamps from individual components. + Must be used with years, months, hours, mins, and secs. + A column that evaluates to an integer. + hours : :class:`~pyspark.sql.Column` or column name, optional + The hour-of-day to represent, from 0 to 23. + Required when creating timestamps from individual components. + Must be used with years, months, days, mins, and secs. + A column that evaluates to an integer. + mins : :class:`~pyspark.sql.Column` or column name, optional + The minute-of-hour to represent, from 0 to 59. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and secs. + A column that evaluates to an integer. + secs : :class:`~pyspark.sql.Column` or column name, optional + The second-of-minute and its micro-fraction to represent, from 0 to 60. + The value can be either an integer like 13, or a fraction like 13.123. + If the sec argument equals to 60, the seconds field is set + to 0 and 1 minute is added to the final timestamp. + Required when creating timestamps from individual components. + Must be used with years, months, days, hours, and mins. + A column that evaluates to a decimal. + date : :class:`~pyspark.sql.Column` or column name, optional + The date to represent, in valid DATE format. + Required when creating timestamps from date and time components. + Must be used with time parameter only. + A column that evaluates to a date. + time : :class:`~pyspark.sql.Column` or column name, optional + The time to represent, in valid TIME format. + Required when creating timestamps from date and time components. + Must be used with date parameter only. + A column that evaluates to a time. Returns ------- :class:`~pyspark.sql.Column` - a string column representing the variant schema - Returns a column that evaluates to a string. + A new column that contains a local date-time, or NULL in case of an error. + Returns a column that evaluates to a timestamp. - Examples + See Also -------- - >>> df = spark.createDataFrame([ {'json': '''{ "a" : 1 }'''} ]) - >>> df.select(schema_of_variant_agg(parse_json(df.json)).alias("r")).collect() - [Row(r='OBJECT')] - """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("schema_of_variant_agg", _to_java_column(v)) - - -# ---------------------- XML Functions ---------------------- - - -@_try_remote_functions -def xpath(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a string array of values within the nodes of xml that match the XPath expression. - - .. versionadded:: 3.5.0 + :meth:`pyspark.sql.functions.make_timestamp` + :meth:`pyspark.sql.functions.make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_timestamp_ntz` + :meth:`pyspark.sql.functions.try_make_timestamp` + :meth:`pyspark.sql.functions.try_make_timestamp_ltz` + :meth:`pyspark.sql.functions.make_time` + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [('b1b2b3c1c2',)], ['x']) - >>> df.select(sf.xpath(df.x, sf.lit('a/b/text()'))).show() - +--------------------+ - |xpath(x, a/b/text())| - +--------------------+ - | [b1, b2, b3]| - +--------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath", xml, path) + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") + Example 1: Make local date-time from years, months, days, hours, mins, secs. -@_try_remote_functions -def xpath_boolean(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns true if the XPath expression evaluates to true, or if a matching node is found. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ntz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |2014-12-28 06:30:45.887 | + +--------------------------------------------------------+ - .. versionadded:: 3.5.0 + Example 2: Make local date-time with invalid input - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('1',)], ['x']) - >>> df.select(sf.xpath_boolean(df.x, sf.lit('a/b'))).show() - +---------------------+ - |xpath_boolean(x, a/b)| - +---------------------+ - | true| - +---------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 13, 28, 6, 30, 45.887]], + ... ['year', 'month', 'day', 'hour', 'min', 'sec']) + >>> df.select( + ... sf.try_make_timestamp_ntz('year', 'month', df.day, df.hour, df.min, df.sec) + ... ).show(truncate=False) + +--------------------------------------------------------+ + |try_make_timestamp_ntz(year, month, day, hour, min, sec)| + +--------------------------------------------------------+ + |NULL | + +--------------------------------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` + >>> spark.conf.unset("spark.sql.session.timeZone") """ - return _invoke_function_over_columns("xpath_boolean", xml, path) + if years is not None: + if any(arg is not None for arg in [date, time]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "try_make_timestamp_ntz", + _ensure_column_or_name(years), + _ensure_column_or_name(months), + _ensure_column_or_name(days), + _ensure_column_or_name(hours), + _ensure_column_or_name(mins), + _ensure_column_or_name(secs), + ) + else: + if any(arg is not None for arg in [years, months, days, hours, mins, secs]): + raise PySparkValueError( + errorClass="CANNOT_SET_TOGETHER", + messageParameters={"arg_list": "years|months|days|hours|mins|secs and date|time"}, + ) + return _invoke_function_over_columns( + "try_make_timestamp_ntz", _ensure_column_or_name(date), _ensure_column_or_name(time) + ) @_try_remote_functions -def xpath_double(xml: "ColumnOrName", path: "ColumnOrName") -> Column: +def make_ym_interval( + years: Optional["ColumnOrName"] = None, + months: Optional["ColumnOrName"] = None, +) -> Column: """ - Returns a double value, the value zero if no match is found, - or NaN if a match is found but the value is non-numeric. + Make year-month interval from years, months. .. versionadded:: 3.5.0 - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_double(df.x, sf.lit('sum(a/b)'))).show() - +-------------------------+ - |xpath_double(x, sum(a/b))| - +-------------------------+ - | 3.0| - +-------------------------+ + Parameters + ---------- + years : :class:`~pyspark.sql.Column` or column name, optional + The number of years, positive or negative. + A column that evaluates to an integer. + months : :class:`~pyspark.sql.Column` or column name, optional + The number of months, positive or negative. + A column that evaluates to an integer. + + Returns + ------- + :class:`~pyspark.sql.Column` + A new column that contains a year-month interval. + Returns a column that evaluates to an interval. See Also -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_double", xml, path) - - -@_try_remote_functions -def xpath_number(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a double value, the value zero if no match is found, - or NaN if a match is found but the value is non-numeric. - - .. versionadded:: 3.5.0 + :meth:`pyspark.sql.functions.make_interval` + :meth:`pyspark.sql.functions.make_dt_interval` + :meth:`pyspark.sql.functions.try_make_interval` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.createDataFrame( - ... [('12',)], ['x'] - ... ).select(sf.xpath_number('x', sf.lit('sum(a/b)'))).show() - +-------------------------+ - |xpath_number(x, sum(a/b))| - +-------------------------+ - | 3.0| - +-------------------------+ + >>> spark.conf.set("spark.sql.session.timeZone", "America/Los_Angeles") - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_number", xml, path) + Example 1: Make year-month interval from years, months. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) + >>> df.select('*', sf.make_ym_interval('year', df.month)).show(truncate=False) + +----+-----+-------------------------------+ + |year|month|make_ym_interval(year, month) | + +----+-----+-------------------------------+ + |2014|12 |INTERVAL '2015-0' YEAR TO MONTH| + +----+-----+-------------------------------+ -@_try_remote_functions -def xpath_float(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a float value, the value zero if no match is found, - or NaN if a match is found but the value is non-numeric. + Example 2: Make year-month interval from years. - .. versionadded:: 3.5.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([[2014, 12]], ['year', 'month']) + >>> df.select('*', sf.make_ym_interval(df.year)).show(truncate=False) + +----+-----+-------------------------------+ + |year|month|make_ym_interval(year, 0) | + +----+-----+-------------------------------+ + |2014|12 |INTERVAL '2014-0' YEAR TO MONTH| + +----+-----+-------------------------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_float(df.x, sf.lit('sum(a/b)'))).show() - +------------------------+ - |xpath_float(x, sum(a/b))| - +------------------------+ - | 3.0| - +------------------------+ + Example 3: Make empty interval. - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.make_ym_interval()).show(truncate=False) + +----------------------------+ + |make_ym_interval(0, 0) | + +----------------------------+ + |INTERVAL '0-0' YEAR TO MONTH| + +----------------------------+ + + >>> spark.conf.unset("spark.sql.session.timeZone") """ - return _invoke_function_over_columns("xpath_float", xml, path) + _years = lit(0) if years is None else years + _months = lit(0) if months is None else months + return _invoke_function_over_columns("make_ym_interval", _years, _months) @_try_remote_functions -def xpath_int(xml: "ColumnOrName", path: "ColumnOrName") -> Column: +def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column: """ - Returns an integer value, or the value zero if no match is found, - or a match is found but the value is non-numeric. + Partition transform function: A transform for any type that partitions + by a hash of the input column. - .. versionadded:: 3.5.0 + .. versionadded:: 3.1.0 + + .. versionchanged:: 3.4.0 + Supports Spark Connect. + + .. deprecated:: 4.0.0 + Use :func:`partitioning.bucket` instead. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_int(df.x, sf.lit('sum(a/b)'))).show() - +----------------------+ - |xpath_int(x, sum(a/b))| - +----------------------+ - | 3| - +----------------------+ + >>> df.writeTo("catalog.db.table").partitionedBy( # doctest: +SKIP + ... bucket(42, "ts") + ... ).createOrReplace() - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_int", xml, path) + Parameters + ---------- + numBuckets : :class:`~pyspark.sql.Column` or int + the number of buckets + col : :class:`~pyspark.sql.Column` or str + target date or timestamp column to work on. + Returns + ------- + :class:`~pyspark.sql.Column` + data partitioned by given columns. + + Notes + ----- + This function can be used only in combination with + :py:meth:`~pyspark.sql.readwriter.DataFrameWriterV2.partitionedBy` + method of the `DataFrameWriterV2`. -@_try_remote_functions -def xpath_long(xml: "ColumnOrName", path: "ColumnOrName") -> Column: """ - Returns a long integer value, or the value zero if no match is found, - or a match is found but the value is non-numeric. + from pyspark.sql.functions import partitioning - .. versionadded:: 3.5.0 + warnings.warn("Deprecated in 4.0.0, use partitioning.bucket instead.", FutureWarning) - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_long(df.x, sf.lit('sum(a/b)'))).show() - +-----------------------+ - |xpath_long(x, sum(a/b))| - +-----------------------+ - | 3| - +-----------------------+ + return partitioning.bucket(numBuckets, col) - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` - """ - return _invoke_function_over_columns("xpath_long", xml, path) + +# Geospatial ST Functions @_try_remote_functions -def xpath_short(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns a short integer value, or the value zero if no match is found, - or a match is found but the value is non-numeric. +def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None) -> Column: + """Returns the input GEOGRAPHY or GEOMETRY value in WKB format. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 + + .. versionchanged:: 4.2.0 + Added the optional `endianness` parameter. + + Parameters + ---------- + geo : :class:`~pyspark.sql.Column` or str + A geospatial value, either a GEOGRAPHY or a GEOMETRY. + endianness : :class:`~pyspark.sql.Column` or str, optional + The optional endianness of the output WKB, 'NDR' for little-endian (default) or 'XDR' for + big-endian. Examples -------- + + Example 1: Getting WKB from GEOGRAPHY. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('12',)], ['x']) - >>> df.select(sf.xpath_short(df.x, sf.lit('sum(a/b)'))).show() - +------------------------+ - |xpath_short(x, sum(a/b))| - +------------------------+ - | 3| - +------------------------+ + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_string` + Example 2: Getting WKB from GEOMETRY. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] + + Example 3: Getting WKB (little-endian) from GEOGRAPHY. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb'), 'NDR'))).collect() + [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] + + Example 4: Getting WKB (big-endian) from GEOMETRY. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb'), 'XDR'))).collect() + [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), XDR))='00000000013FF00000000000004000000000000000')] """ - return _invoke_function_over_columns("xpath_short", xml, path) + if endianness is None: + return _invoke_function_over_columns("st_asbinary", geo) + else: + _endianness = lit(endianness) if isinstance(endianness, str) else endianness + return _invoke_function_over_columns("st_asbinary", geo, _endianness) @_try_remote_functions -def xpath_string(xml: "ColumnOrName", path: "ColumnOrName") -> Column: - """ - Returns the text contents of the first xml node that matches the XPath expression. +def st_geogfromwkb(wkb: "ColumnOrName") -> Column: + """Parses the input WKB description and returns the corresponding GEOGRAPHY value. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 + + Parameters + ---------- + wkb : :class:`~pyspark.sql.Column` or str + A BINARY value in WKB format, representing a GEOGRAPHY value. + A column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([('bcc',)], ['x']) - >>> df.select(sf.xpath_string(df.x, sf.lit('a/c'))).show() - +--------------------+ - |xpath_string(x, a/c)| - +--------------------+ - | cc| - +--------------------+ - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] """ - return _invoke_function_over_columns("xpath_string", xml, path) + return _invoke_function_over_columns("st_geogfromwkb", wkb) -# TODO: Fix and add an example for StructType with Spark Connect -# e.g., StructType([StructField("a", IntegerType())]) @_try_remote_functions -def from_xml( - col: "ColumnOrName", - schema: Union[StructType, Column, str], - options: Optional[Mapping[str, str]] = None, +def st_geomfromwkb( + wkb: "ColumnOrName", srid: Optional[Union["ColumnOrName", int]] = None ) -> Column: - """ - Parses a column containing a XML string to a row with - the specified schema. Returns `null`, in the case of an unparsable string. + """Parses the input WKB description and returns the corresponding GEOMETRY value. - .. versionadded:: 4.0.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a column or column name in XML format - schema : :class:`StructType`, :class:`~pyspark.sql.Column` or str - a StructType, Column or Python string literal with a DDL-formatted string - A column that evaluates to a string, or a DDL-formatted type string, or a DataType. - to use when parsing the Xml column - options : dict, optional - options to control parsing. accepts the same options as the Xml datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa - - Returns - ------- - :class:`~pyspark.sql.Column` - a new column of complex type from given XML object. - Returns a column that evaluates to a struct. + wkb : :class:`~pyspark.sql.Column` or str + A BINARY value in WKB format, representing a GEOMETRY value. + A column that evaluates to a binary. + srid : :class:`~pyspark.sql.Column` or int, optional + The optional SRID value of the geometry. Default is 0. + A column that evaluates to an integer. Examples -------- - Example 1: Parsing XML with a DDL-formatted string schema - - >>> import pyspark.sql.functions as sf - >>> data = [(1, '''

1

''')] - >>> df = spark.createDataFrame(data, ("key", "value")) - ... # Define the schema using a DDL-formatted string - >>> schema = "STRUCT" - ... # Parse the XML column using the DDL-formatted schema - >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() - [Row(xml=Row(a=1))] - - Example 2: Parsing XML with a :class:`StructType` schema - - >>> import pyspark.sql.functions as sf - >>> from pyspark.sql.types import StructType, LongType - >>> data = [(1, '''

1

''')] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> schema = StructType().add("a", LongType()) - >>> df.select(sf.from_xml(df.value, schema)).show() - +---------------+ - |from_xml(value)| - +---------------+ - | {1}| - +---------------+ - - Example 3: Parsing XML with :class:`ArrayType` in schema - - >>> import pyspark.sql.functions as sf - >>> data = [(1, '

12

')] - >>> df = spark.createDataFrame(data, ("key", "value")) - ... # Define the schema with an Array type - >>> schema = "STRUCT>" - ... # Parse the XML column using the schema with an Array - >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() - [Row(xml=Row(a=[1, 2]))] - - Example 4: Parsing XML using :meth:`pyspark.sql.functions.schema_of_xml` - - >>> import pyspark.sql.functions as sf - >>> # Sample data with an XML column - ... data = [(1, '

12

')] - >>> df = spark.createDataFrame(data, ("key", "value")) - ... # Generate the schema from an example XML value - >>> schema = sf.schema_of_xml(sf.lit(data[0][1])) - ... # Parse the XML column using the generated schema - >>> df.select(sf.from_xml(df.value, schema).alias("xml")).collect() - [Row(xml=Row(a=[1, 2]))] - - See Also - -------- - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() + [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] """ - from pyspark.sql.classic.column import _to_java_column - - if isinstance(schema, StructType): - schema = schema.json() - elif isinstance(schema, Column): - schema = _to_java_column(schema) - elif not isinstance(schema, str): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "StructType, Column or str", - "arg_name": "schema", - "arg_type": type(schema).__name__, - }, - ) - return _invoke_function("from_xml", _to_java_column(col), schema, _options_to_str(options)) + if srid is None: + return _invoke_function_over_columns("st_geomfromwkb", wkb) + else: + srid = _enum_to_value(srid) + srid = lit(srid) if isinstance(srid, int) else srid + return _invoke_function_over_columns("st_geomfromwkb", wkb, srid) @_try_remote_functions -def schema_of_xml(xml: Union[Column, str], options: Optional[Mapping[str, str]] = None) -> Column: - """ - Parses a XML string and infers its schema in DDL format. +def st_setsrid(geo: "ColumnOrName", srid: Union["ColumnOrName", int]) -> Column: + """Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. - .. versionadded:: 4.0.0 + .. versionadded:: 4.1.0 Parameters ---------- - xml : :class:`~pyspark.sql.Column` or str - A column that evaluates to a string. - a XML string or a foldable string column containing a XML string. - options : dict, optional - options to control parsing. accepts the same options as the XML datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa - - Returns - ------- - :class:`~pyspark.sql.Column` - a string representation of a :class:`StructType` parsed from given XML. - Returns a column that evaluates to a string. - - Examples - -------- - Example 1: Parsing a simple XML with a single element + geo : :class:`~pyspark.sql.Column` or str + A geospatial value, either a GEOGRAPHY or a GEOMETRY. + srid : :class:`~pyspark.sql.Column` or int + An INTEGER representing the new SRID of the geospatial value. - >>> from pyspark.sql import functions as sf - >>> df = spark.range(1) - >>> df.select(sf.schema_of_xml(sf.lit('

1

')).alias("xml")).collect() - [Row(xml='STRUCT')] + Examples + -------- - Example 2: Parsing an XML with multiple elements in an array + Example 1: Setting the SRID on GEOGRAPHY with SRID from another column. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'), 4326)], ['wkb', 'srid']) # noqa + >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geogfromwkb('wkb'), 'srid'))).collect() + [Row(st_srid(st_setsrid(st_geogfromwkb(wkb), srid))=4326)] + Example 2: Setting the SRID on GEOMETRY with SRID as an integer literal. >>> from pyspark.sql import functions as sf - >>> df.select(sf.schema_of_xml(sf.lit('

12

')).alias("xml")).collect() - [Row(xml='STRUCT>')] + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geomfromwkb('wkb'), 4326))).collect() + [Row(st_srid(st_setsrid(st_geomfromwkb(wkb, 0), 4326))=4326)] + """ + srid = _enum_to_value(srid) + srid = lit(srid) if isinstance(srid, int) else srid + return _invoke_function_over_columns("st_setsrid", geo, srid) - Example 3: Parsing XML with options to exclude attributes - >>> from pyspark.sql import functions as sf - >>> schema = sf.schema_of_xml('

1

', {'excludeAttribute':'true'}) - >>> df.select(schema.alias("xml")).collect() - [Row(xml='STRUCT')] +@_try_remote_functions +def st_srid(geo: "ColumnOrName") -> Column: + """Returns the SRID of the input GEOGRAPHY or GEOMETRY value. - Example 4: Parsing XML with complex structure + .. versionadded:: 4.1.0 - >>> from pyspark.sql import functions as sf - >>> df.select( - ... sf.schema_of_xml( - ... sf.lit('Alice30') - ... ).alias("xml") - ... ).collect() - [Row(xml='STRUCT>')] + Parameters + ---------- + geo : :class:`~pyspark.sql.Column` or str + A geospatial value, either a GEOGRAPHY or a GEOMETRY. - Example 5: Parsing XML with nested arrays + Examples + -------- + Example 1: Getting the SRID of GEOGRAPHY. >>> from pyspark.sql import functions as sf - >>> df.select( - ... sf.schema_of_xml( - ... sf.lit('12') - ... ).alias("xml") - ... ).collect() - [Row(xml='STRUCT>>')] + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.st_srid(sf.st_geogfromwkb('wkb'))).collect() + [Row(st_srid(st_geogfromwkb(wkb))=4326)] - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.to_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` + Example 2: Getting the SRID of GEOMETRY. + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa + >>> df.select(sf.st_srid(sf.st_geomfromwkb('wkb'))).collect() + [Row(st_srid(st_geomfromwkb(wkb, 0))=0)] """ - from pyspark.sql.classic.column import _to_java_column + return _invoke_function_over_columns("st_srid", geo) - xml = _enum_to_value(xml) - if not isinstance(xml, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "xml", - "arg_type": type(xml).__name__, - }, - ) - return _invoke_function("schema_of_xml", _to_java_column(lit(xml)), _options_to_str(options)) +# Call Functions @_try_remote_functions -def to_xml(col: "ColumnOrName", options: Optional[Mapping[str, str]] = None) -> Column: +def call_udf(udfName: str, *cols: "ColumnOrName") -> Column: """ - Converts a column containing a :class:`StructType` into a XML string. - Throws an exception, in the case of an unsupported type. + Call a user-defined function. - .. versionadded:: 4.0.0 + .. versionadded:: 3.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or str - A column that evaluates to a struct, array, map, or variant. - name of column containing a struct. - options: dict, optional - options to control converting. accepts the same options as the XML datasource. - See `Data Source Option `_ - A dict of options. Each key and value is a string. - for the version you use. - - .. # noqa + udfName : str + name of the user defined function (UDF) + cols : :class:`~pyspark.sql.Column` or str + column names or :class:`~pyspark.sql.Column`\\s to be used in the UDF Returns ------- :class:`~pyspark.sql.Column` - a XML string converted from given :class:`StructType`. - Returns a column that evaluates to a string. + result of executed udf. Examples -------- - >>> from pyspark.sql import Row - >>> data = [(1, Row(age=2, name='Alice'))] - >>> df = spark.createDataFrame(data, ("key", "value")) - >>> df.select(to_xml(df.value, {'rowTag':'person'}).alias("xml")).collect() - [Row(xml='\\n 2\\n Alice\\n')] - - See Also - -------- - :meth:`pyspark.sql.functions.from_xml` - :meth:`pyspark.sql.functions.schema_of_xml` - :meth:`pyspark.sql.functions.xpath` - :meth:`pyspark.sql.functions.xpath_boolean` - :meth:`pyspark.sql.functions.xpath_double` - :meth:`pyspark.sql.functions.xpath_float` - :meth:`pyspark.sql.functions.xpath_int` - :meth:`pyspark.sql.functions.xpath_long` - :meth:`pyspark.sql.functions.xpath_number` - :meth:`pyspark.sql.functions.xpath_short` - :meth:`pyspark.sql.functions.xpath_string` + >>> from pyspark.sql.functions import call_udf, col + >>> from pyspark.sql.types import IntegerType, StringType + >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) + >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) + >>> df.select(call_udf("intX2", "id")).show() + +---------+ + |intX2(id)| + +---------+ + | 2| + | 4| + | 6| + +---------+ + >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) + >>> df.select(call_udf("strX2", col("name"))).show() + +-----------+ + |strX2(name)| + +-----------+ + | aa| + | bb| + | cc| + +-----------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("to_xml", _to_java_column(col), _options_to_str(options)) - + from pyspark.sql.classic.column import _to_java_column, _to_seq -# ---------------------- URL Functions ---------------------- + sc = _get_active_spark_context() + return _invoke_function("call_udf", udfName, _to_seq(sc, cols, _to_java_column)) @_try_remote_functions -def try_parse_url( - url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None -) -> Column: +def call_function(funcName: str, *cols: "ColumnOrName") -> Column: """ - This is a special version of `parse_url` that performs the same operation, but returns a - NULL value instead of raising an error if the parsing cannot be performed. + Call a SQL function. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - url : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL. - A column that evaluates to a string. - partToExtract : :class:`~pyspark.sql.Column` or str - A column of strings, each representing the part to extract from the URL. - A column that evaluates to a string. - key : :class:`~pyspark.sql.Column` or str, optional - A column of strings, each representing the key of a query parameter in the URL. - A column that evaluates to a string. + funcName : str + function name that follows the SQL identifier syntax (can be quoted, can be qualified) + cols : :class:`~pyspark.sql.Column` or str + column names or :class:`~pyspark.sql.Column`\\s to be used in the function Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the value of the extracted part from the URL. - Returns a column that evaluates to a string. + result of executed function. Examples -------- - Example 1: Extracting the query part from a URL + >>> from pyspark.sql.functions import call_udf, col + >>> from pyspark.sql.types import IntegerType, StringType + >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) + >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) + >>> df.select(call_function("intX2", "id")).show() + +---------+ + |intX2(id)| + +---------+ + | 2| + | 4| + | 6| + +---------+ + >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) + >>> df.select(call_function("strX2", col("name"))).show() + +-----------+ + |strX2(name)| + +-----------+ + | aa| + | bb| + | cc| + +-----------+ + >>> df.select(call_function("avg", col("id"))).show() + +-------+ + |avg(id)| + +-------+ + | 2.0| + +-------+ + >>> _ = spark.sql("CREATE FUNCTION custom_avg AS 'test.org.apache.spark.sql.MyDoubleAvg'") + ... # doctest: +SKIP + >>> df.select(call_function("custom_avg", col("id"))).show() + ... # doctest: +SKIP + +------------------------------------+ + |spark_catalog.default.custom_avg(id)| + +------------------------------------+ + | 102.0| + +------------------------------------+ + >>> df.select(call_function("spark_catalog.default.custom_avg", col("id"))).show() + ... # doctest: +SKIP + +------------------------------------+ + |spark_catalog.default.custom_avg(id)| + +------------------------------------+ + | 102.0| + +------------------------------------+ + """ + from pyspark.sql.classic.column import _to_java_column, _to_seq - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | query=1| - +------------------------+ + sc = _get_active_spark_context() + return _invoke_function("call_function", funcName, _to_seq(sc, cols, _to_java_column)) - Example 2: Extracting the value of a specific query parameter from a URL - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], - ... ["url", "part", "key"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() - +-----------------------------+ - |try_parse_url(url, part, key)| - +-----------------------------+ - | 1| - +-----------------------------+ +@_try_remote_functions +def unwrap_udt(col: "ColumnOrName") -> Column: + """ + Unwrap UDT data type column into its underlying type. - Example 3: Extracting the protocol part from a URL + .. versionadded:: 3.4.0 - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | https| - +------------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name - Example 4: Extracting the host part from a URL + Returns + ------- + :class:`~pyspark.sql.Column` + The underlying representation. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "HOST")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | spark.apache.org| - +------------------------+ + See Also + -------- + :meth:`pyspark.sql.functions.wrap_udt` - Example 5: Extracting the path part from a URL + Examples + -------- + Example 1: Unwrap ML-specific UDT - VectorUDT >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PATH")], - ... ["url", "part"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part)).show() - +------------------------+ - |try_parse_url(url, part)| - +------------------------+ - | /path| - +------------------------+ + >>> from pyspark.ml.linalg import Vectors + >>> vec1 = Vectors.dense(1, 2, 3) + >>> vec2 = Vectors.sparse(4, {1: 1.0, 3: 5.5}) + >>> df = spark.createDataFrame([(vec1,), (vec2,)], ["vec"]) + >>> df.select(sf.unwrap_udt("vec")).printSchema() + root + |-- unwrap_udt(vec): struct (nullable = true) + | |-- type: byte (nullable = false) + | |-- size: integer (nullable = true) + | |-- indices: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- values: array (nullable = true) + | | |-- element: double (containsNull = false) - Example 6: Invalid URL + Example 2: Unwrap ML-specific UDT - MatrixUDT >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("inva lid://spark.apache.org/path?query=1", "QUERY", "query")], - ... ["url", "part", "key"] - ... ) - >>> df.select(sf.try_parse_url(df.url, df.part, df.key)).show() - +-----------------------------+ - |try_parse_url(url, part, key)| - +-----------------------------+ - | NULL| - +-----------------------------+ + >>> from pyspark.ml.linalg import Matrices + >>> mat1 = Matrices.dense(2, 2, range(4)) + >>> mat2 = Matrices.sparse(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4]) + >>> df = spark.createDataFrame([(mat1,), (mat2,)], ["mat"]) + >>> df.select(sf.unwrap_udt("mat")).printSchema() + root + |-- unwrap_udt(mat): struct (nullable = true) + | |-- type: byte (nullable = false) + | |-- numRows: integer (nullable = false) + | |-- numCols: integer (nullable = false) + | |-- colPtrs: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- rowIndices: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- values: array (nullable = true) + | | |-- element: double (containsNull = false) + | |-- isTransposed: boolean (nullable = false) """ - if key is not None: - return _invoke_function_over_columns("try_parse_url", url, partToExtract, key) - else: - return _invoke_function_over_columns("try_parse_url", url, partToExtract) + from pyspark.sql.classic.column import _to_java_column + + return _invoke_function("unwrap_udt", _to_java_column(col)) @_try_remote_functions -def parse_url( - url: "ColumnOrName", partToExtract: "ColumnOrName", key: Optional["ColumnOrName"] = None -) -> Column: +def wrap_udt(col: "ColumnOrName", udt: "Union[UserDefinedType, Column]") -> Column: """ - URL function: Extracts a specified part from a URL. If a key is provided, - it returns the associated query parameter value. + Wrap a column as a user-defined type. - .. versionadded:: 3.5.0 + .. versionadded:: 4.4.0 Parameters ---------- - url : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL. - A column that evaluates to a string. - partToExtract : :class:`~pyspark.sql.Column` or str - A column of strings, each representing the part to extract from the URL. - A column that evaluates to a string. - key : :class:`~pyspark.sql.Column` or str, optional - A column of strings, each representing the key of a query parameter in the URL. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The column to wrap. The column data type must match the UDT's underlying SQL type. + udt : :class:`~pyspark.sql.types.UserDefinedType` or :class:`~pyspark.sql.Column` + The target user-defined type, or a constant string column containing its JSON + representation. Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the value of the extracted part from the URL. - Returns a column that evaluates to a string. + A column of the target user-defined type. - Examples + See Also -------- - Example 1: Extracting the query part from a URL - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | query=1| - +--------------------+ - - Example 2: Extracting the value of a specific query parameter from a URL - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "QUERY", "query")], - ... ["url", "part", "key"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part, df.key)).show() - +-------------------------+ - |parse_url(url, part, key)| - +-------------------------+ - | 1| - +-------------------------+ - - Example 3: Extracting the protocol part from a URL - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PROTOCOL")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | https| - +--------------------+ + :meth:`pyspark.sql.functions.unwrap_udt` - Example 4: Extracting the host part from a URL + Examples + -------- + Example 1: Wrapping a vector struct as VectorUDT >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Row + >>> from pyspark.sql.types import StructField, StructType + >>> from pyspark.ml.linalg import VectorUDT + >>> vector_schema = StructType([StructField("vec", VectorUDT.sqlType(), True)]) >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "HOST")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | spark.apache.org| - +--------------------+ + ... [(Row(type=1, size=None, indices=None, values=[1.0, 2.0, 3.0]),)], + ... vector_schema) + >>> df.select("*", sf.wrap_udt("vec", VectorUDT())).show() + +--------------------+...+ + | vec|wrap_udt(vec...| + +--------------------+...+ + |{1, NULL, NULL, [...|...[1.0,2.0,3.0]| + +--------------------+...+ + >>> row = df.select(sf.wrap_udt("vec", VectorUDT())).first() + >>> type(row[0]) + - Example 5: Extracting the path part from a URL + Example 2: Wrapping a matrix struct as MatrixUDT >>> from pyspark.sql import functions as sf + >>> from pyspark.sql import Row + >>> from pyspark.sql.types import StructField, StructType + >>> from pyspark.mllib.linalg import MatrixUDT + >>> matrix_schema = StructType([StructField("mat", MatrixUDT.sqlType(), True)]) >>> df = spark.createDataFrame( - ... [("https://spark.apache.org/path?query=1", "PATH")], - ... ["url", "part"] - ... ) - >>> df.select(sf.parse_url(df.url, df.part)).show() - +--------------------+ - |parse_url(url, part)| - +--------------------+ - | /path| - +--------------------+ + ... [( + ... Row( + ... type=1, + ... numRows=2, + ... numCols=2, + ... colPtrs=None, + ... rowIndices=None, + ... values=[1.0, 2.0, 3.0, 4.0], + ... isTransposed=False), + ... )], + ... matrix_schema) + >>> df.select("*", sf.wrap_udt("mat", MatrixUDT())).printSchema() + root + |-- mat: struct (nullable = true) + | |-- type: byte (nullable = false) + | |-- numRows: integer (nullable = false) + | |-- numCols: integer (nullable = false) + | |-- colPtrs: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- rowIndices: array (nullable = true) + | | |-- element: integer (containsNull = false) + | |-- values: array (nullable = true) + | | |-- element: double (containsNull = false) + | |-- isTransposed: boolean (nullable = false) + |-- wrap_udt(mat...: matrix... (nullable = true) + >>> row = df.select(sf.wrap_udt("mat", MatrixUDT())).first() + >>> type(row[0]) + """ - if key is not None: - return _invoke_function_over_columns("parse_url", url, partToExtract, key) + from pyspark.sql.classic.column import _to_java_column + + if isinstance(udt, _UserDefinedType): + udt_col = lit(udt.json()) + elif isinstance(udt, Column): + udt_col = udt else: - return _invoke_function_over_columns("parse_url", url, partToExtract) + raise PySparkTypeError( + errorClass="NOT_EXPECTED_TYPE", + messageParameters={ + "expected_type": "UserDefinedType or Column", + "arg_name": "udt", + "arg_type": type(udt).__name__, + }, + ) + return _invoke_function("wrap_udt", _to_java_column(col), _to_java_column(udt_col)) + + +# ---------------------- Datasketch functions ------------------------------ @_try_remote_functions -def url_decode(str: "ColumnOrName") -> Column: +def hll_sketch_agg( + col: "ColumnOrName", + lgConfigK: Optional[Union[int, Column]] = None, +) -> Column: """ - URL function: Decodes a URL-encoded string in 'application/x-www-form-urlencoded' - format to its original format. + Aggregate function: returns the updatable binary representation of the Datasketches + HllSketch configured with lgConfigK arg. .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL-encoded string. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an integer, long, string, or binary. + lgConfigK : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of K, where K is the number of buckets or slots for the HllSketch. + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the decoded string. - Returns a column that evaluates to a string. + The binary representation of the HllSketch. - Examples + See Also -------- - Example 1: Decoding a URL-encoded string - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show(truncate=False) - +------------------------+ - |url_decode(url) | - +------------------------+ - |https://spark.apache.org| - +------------------------+ - - Example 2: Decoding a URL-encoded string with spaces - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Hello%20World%21",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | Hello World!| - +---------------+ - - Example 3: Decoding a URL-encoded string with special characters - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("A%2BB%3D%3D",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | A+B==| - +---------------+ - - Example 4: Decoding a URL-encoded string with non-ASCII characters + :meth:`pyspark.sql.functions.hll_union` + :meth:`pyspark.sql.functions.hll_union_agg` + :meth:`pyspark.sql.functions.hll_sketch_estimate` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("%E4%BD%A0%E5%A5%BD",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | 你好| - +---------------+ - - Example 5: Decoding a URL-encoded string with hexadecimal values + >>> df = spark.createDataFrame([1,2,2,3], "INT") + >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() + +----------------------------------------------+ + |hll_sketch_estimate(hll_sketch_agg(value, 12))| + +----------------------------------------------+ + | 3| + +----------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("%7E%21%40%23%24%25%5E%26%2A%28%29%5F%2B",)], ["url"]) - >>> df.select(sf.url_decode(df.url)).show() - +---------------+ - |url_decode(url)| - +---------------+ - | ~!@#$%^&*()_+| - +---------------+ + >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value", 12))).show() + +----------------------------------------------+ + |hll_sketch_estimate(hll_sketch_agg(value, 12))| + +----------------------------------------------+ + | 3| + +----------------------------------------------+ """ - return _invoke_function_over_columns("url_decode", str) + if lgConfigK is None: + return _invoke_function_over_columns("hll_sketch_agg", col) + else: + return _invoke_function_over_columns("hll_sketch_agg", col, lit(lgConfigK)) @_try_remote_functions -def try_url_decode(str: "ColumnOrName") -> Column: +def hll_union_agg( + col: "ColumnOrName", + allowDifferentLgConfigK: Optional[Union[bool, Column]] = None, +) -> Column: """ - This is a special version of `url_decode` that performs the same operation, but returns a - NULL value instead of raising an error if the decoding cannot be performed. + Aggregate function: returns the updatable binary representation of the Datasketches + HllSketch, generated by merging previously created Datasketches HllSketch instances + via a Datasketches Union instance. Throws an exception if sketches have different + lgConfigK values and allowDifferentLgConfigK is unset or set to false. - .. versionadded:: 4.0.0 + .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a URL-encoded string. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + allowDifferentLgConfigK : :class:`~pyspark.sql.Column` or bool, optional + Allow sketches with different lgConfigK values to be merged (defaults to false). Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the decoded string. - Returns a column that evaluates to a string. + The binary representation of the merged HllSketch. - Examples + See Also -------- - Example 1: Decoding a URL-encoded string + :meth:`pyspark.sql.functions.hll_union` + :meth:`pyspark.sql.functions.hll_sketch_agg` + :meth:`pyspark.sql.functions.hll_sketch_estimate` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https%3A%2F%2Fspark.apache.org",)], ["url"]) - >>> df.select(sf.try_url_decode(df.url)).show(truncate=False) - +------------------------+ - |try_url_decode(url) | - +------------------------+ - |https://spark.apache.org| - +------------------------+ - - Example 2: Return NULL if the decoding cannot be performed. + >>> df1 = spark.createDataFrame([1,2,2,3], "INT") + >>> df1 = df1.agg(sf.hll_sketch_agg("value").alias("sketch")) + >>> df2 = spark.createDataFrame([4,5,5,6], "INT") + >>> df2 = df2.agg(sf.hll_sketch_agg("value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch"))).show() + +-------------------------------------------------+ + |hll_sketch_estimate(hll_union_agg(sketch, false))| + +-------------------------------------------------+ + | 6| + +-------------------------------------------------+ - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https%3A%2F%2spark.apache.org",)], ["url"]) - >>> df.select(sf.try_url_decode(df.url)).show() - +-------------------+ - |try_url_decode(url)| - +-------------------+ - | NULL| - +-------------------+ + >>> df3.agg(sf.hll_sketch_estimate(sf.hll_union_agg("sketch", False))).show() + +-------------------------------------------------+ + |hll_sketch_estimate(hll_union_agg(sketch, false))| + +-------------------------------------------------+ + | 6| + +-------------------------------------------------+ """ - return _invoke_function_over_columns("try_url_decode", str) + if allowDifferentLgConfigK is None: + return _invoke_function_over_columns("hll_union_agg", col) + else: + return _invoke_function_over_columns("hll_union_agg", col, lit(allowDifferentLgConfigK)) @_try_remote_functions -def url_encode(str: "ColumnOrName") -> Column: +def hll_sketch_estimate(col: "ColumnOrName") -> Column: """ - URL function: Encodes a string into a URL-encoded string in - 'application/x-www-form-urlencoded' format. + Returns the estimated number of unique values given the binary representation + of a Datasketches HllSketch. .. versionadded:: 3.5.0 Parameters ---------- - str : :class:`~pyspark.sql.Column` or str - A column of strings, each representing a string to be URL-encoded. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name Returns ------- :class:`~pyspark.sql.Column` - A new column of strings, each representing the URL-encoded string. - Returns a column that evaluates to a string. + The estimated number of unique values for the HllSketch. - Examples + See Also -------- - Example 1: Encoding a simple URL - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("https://spark.apache.org",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show(truncate=False) - +------------------------------+ - |url_encode(url) | - +------------------------------+ - |https%3A%2F%2Fspark.apache.org| - +------------------------------+ - - Example 2: Encoding a URL with spaces - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("Hello World!",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show() - +---------------+ - |url_encode(url)| - +---------------+ - | Hello+World%21| - +---------------+ - - Example 3: Encoding a URL with special characters - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("A+B==",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show() - +---------------+ - |url_encode(url)| - +---------------+ - | A%2BB%3D%3D| - +---------------+ - - Example 4: Encoding a URL with non-ASCII characters - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("你好",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show() - +------------------+ - | url_encode(url)| - +------------------+ - |%E4%BD%A0%E5%A5%BD| - +------------------+ - - Example 5: Encoding a URL with hexadecimal values + :meth:`pyspark.sql.functions.hll_union` + :meth:`pyspark.sql.functions.hll_union_agg` + :meth:`pyspark.sql.functions.hll_sketch_agg` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("~!@#$%^&*()_+",)], ["url"]) - >>> df.select(sf.url_encode(df.url)).show(truncate=False) - +-----------------------------------+ - |url_encode(url) | - +-----------------------------------+ - |%7E%21%40%23%24%25%5E%26*%28%29_%2B| - +-----------------------------------+ + >>> df = spark.createDataFrame([1,2,2,3], "INT") + >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() + +----------------------------------------------+ + |hll_sketch_estimate(hll_sketch_agg(value, 12))| + +----------------------------------------------+ + | 3| + +----------------------------------------------+ """ - return _invoke_function_over_columns("url_encode", str) - + from pyspark.sql.classic.column import _to_java_column -# ---------------------- Misc Functions ---------------------- + return _invoke_function("hll_sketch_estimate", _to_java_column(col)) @_try_remote_functions -def input_file_name() -> Column: +def hll_union( + col1: "ColumnOrName", col2: "ColumnOrName", allowDifferentLgConfigK: Optional[bool] = None +) -> Column: """ - Creates a string column for the file name of the current Spark task. + Merges two binary representations of Datasketches HllSketch objects, using a + Datasketches Union object. Throws an exception if sketches have different + lgConfigK values and allowDifferentLgConfigK is unset or set to false. - .. versionadded:: 1.6.0 + .. versionadded:: 3.5.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + col2 : :class:`~pyspark.sql.Column` or column name + allowDifferentLgConfigK : bool, optional + Allow sketches with different lgConfigK values to be merged (defaults to false). Returns ------- :class:`~pyspark.sql.Column` - file names. + The binary representation of the merged HllSketch. See Also -------- - :meth:`pyspark.sql.functions.input_file_block_length` - :meth:`pyspark.sql.functions.input_file_block_start` + :meth:`pyspark.sql.functions.hll_union_agg` + :meth:`pyspark.sql.functions.hll_sketch_agg` + :meth:`pyspark.sql.functions.hll_sketch_estimate` Examples -------- - >>> import os >>> from pyspark.sql import functions as sf - >>> path = os.path.abspath(__file__) - >>> df = spark.read.text(path) - >>> df.select(sf.input_file_name()).first() - Row(input_file_name()='file:///...') + >>> df = spark.createDataFrame([(1,4),(2,5),(2,5),(3,6)], "struct") + >>> df = df.agg( + ... sf.hll_sketch_agg("v1").alias("sketch1"), + ... sf.hll_sketch_agg("v2").alias("sketch2") + ... ) + >>> df.select(sf.hll_sketch_estimate(sf.hll_union(df.sketch1, "sketch2"))).show() + +-------------------------------------------------------+ + |hll_sketch_estimate(hll_union(sketch1, sketch2, false))| + +-------------------------------------------------------+ + | 6| + +-------------------------------------------------------+ """ - return _invoke_function("input_file_name") - - -@_try_remote_functions -def monotonically_increasing_id() -> Column: - """A column that generates monotonically increasing 64-bit integers. + from pyspark.sql.classic.column import _to_java_column - The generated ID is guaranteed to be monotonically increasing and unique, but not consecutive. - The current implementation puts the partition ID in the upper 31 bits, and the record number - within each partition in the lower 33 bits. The assumption is that the data frame has - less than 1 billion partitions, and each partition has less than 8 billion records. + if allowDifferentLgConfigK is not None: + return _invoke_function( + "hll_union", + _to_java_column(col1), + _to_java_column(col2), + _enum_to_value(allowDifferentLgConfigK), + ) + else: + return _invoke_function("hll_union", _to_java_column(col1), _to_java_column(col2)) - .. versionadded:: 1.6.0 - .. versionchanged:: 3.4.0 - Supports Spark Connect. +@_try_remote_functions +def theta_sketch_agg( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + ThetaSketch with the values in the input column configured with lgNomEntries nominal entries. - Notes - ----- - The function is non-deterministic because its result depends on partition IDs. + .. versionadded:: 4.1.0 - As an example, consider a :class:`DataFrame` with two partitions, each with 3 records. - This expression would return the following IDs: - 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to an array, binary, double, float, integer, long, or string. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries, where nominal entries is the size of the sketch + (must be between 4 and 26, defaults to 12). + A column that evaluates to an integer. Returns ------- :class:`~pyspark.sql.Column` - last value of the group. + The binary representation of the ThetaSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.theta_difference` + :meth:`pyspark.sql.functions.theta_union_agg` + :meth:`pyspark.sql.functions.theta_intersection_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(0, 10, 1, 2).select( - ... "*", - ... sf.spark_partition_id(), - ... sf.monotonically_increasing_id()).show() - +---+--------------------+-----------------------------+ - | id|SPARK_PARTITION_ID()|monotonically_increasing_id()| - +---+--------------------+-----------------------------+ - | 0| 0| 0| - | 1| 0| 1| - | 2| 0| 2| - | 3| 0| 3| - | 4| 0| 4| - | 5| 1| 8589934592| - | 6| 1| 8589934593| - | 7| 1| 8589934594| - | 8| 1| 8589934595| - | 9| 1| 8589934596| - +---+--------------------+-----------------------------+ + >>> df = spark.createDataFrame([1,2,2,3], "INT") + >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_sketch_agg(value, 12))| + +--------------------------------------------------+ + | 3| + +--------------------------------------------------+ + + >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value", 15))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_sketch_agg(value, 15))| + +--------------------------------------------------+ + | 3| + +--------------------------------------------------+ """ - return _invoke_function("monotonically_increasing_id") + fn = "theta_sketch_agg" + if lgNomEntries is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) @_try_remote_functions -def spark_partition_id() -> Column: - """A column for partition ID. - - .. versionadded:: 1.6.0 +def theta_union_agg( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + ThetaSketch that is the union of the Theta sketches in the input column. - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 - Notes - ----- - This is non deterministic because it depends on data partitioning and task scheduling. + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries for the union operation + (must be between 4 and 26, defaults to 12) Returns ------- :class:`~pyspark.sql.Column` - partition id the record belongs to. - - Examples - -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(10, numPartitions=5).select("*", sf.spark_partition_id()).show() - +---+--------------------+ - | id|SPARK_PARTITION_ID()| - +---+--------------------+ - | 0| 0| - | 1| 0| - | 2| 1| - | 3| 1| - | 4| 2| - | 5| 2| - | 6| 3| - | 7| 3| - | 8| 4| - | 9| 4| - +---+--------------------+ - """ - return _invoke_function("spark_partition_id") - - -@_try_remote_functions -def current_catalog() -> Column: - """Returns the current catalog. - - .. versionadded:: 3.5.0 + The binary representation of the merged ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.current_database` - :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_catalog()).show() - +-----------------+ - |current_catalog()| - +-----------------+ - | spark_catalog| - +-----------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1,2,2,3], "INT") + >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df2 = spark.createDataFrame([4,5,5,6], "INT") + >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.theta_sketch_estimate(sf.theta_union_agg("sketch"))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_union_agg(sketch, 12))| + +--------------------------------------------------+ + | 6| + +--------------------------------------------------+ """ - return _invoke_function("current_catalog") + fn = "theta_union_agg" + if lgNomEntries is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(lgNomEntries)) @_try_remote_functions -def current_path() -> Column: - """Returns the current SQL path as a comma-separated list of qualified schema names. +def theta_intersection_agg(col: "ColumnOrName") -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + ThetaSketch that is the intersection of the Theta sketches in the input column - .. versionadded:: 4.2.0 + .. versionadded:: 4.1.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the intersected ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_database` - :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_path()).show() # doctest: +SKIP - +----------------------------------------------------+ - | current_path()| - +----------------------------------------------------+ - |system.builtin,system.session,spark_catalog.default | - +----------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1,2,2,3], "INT") + >>> df1 = df1.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df2 = spark.createDataFrame([2,3,3,4], "INT") + >>> df2 = df2.agg(sf.theta_sketch_agg("value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.theta_sketch_estimate(sf.theta_intersection_agg("sketch"))).show() + +-----------------------------------------------------+ + |theta_sketch_estimate(theta_intersection_agg(sketch))| + +-----------------------------------------------------+ + | 2| + +-----------------------------------------------------+ """ - return _invoke_function("current_path") + fn = "theta_intersection_agg" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def current_database() -> Column: - """Returns the current database. +def tuple_sketch_agg_double( + key: "ColumnOrName", + summary: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch with double summaries built from the key and summary columns. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 + + Parameters + ---------- + key : :class:`~pyspark.sql.Column` or column name + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. + summary : :class:`~pyspark.sql.Column` or column name + The column containing double summary values. + A column that evaluates to a double. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_schema` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` + :meth:`pyspark.sql.functions.tuple_sketch_summary_double` + :meth:`pyspark.sql.functions.tuple_union_agg_double` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_database()).show() - +----------------+ - |current_schema()| - +----------------+ - | default| - +----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +--------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| + +--------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------+ """ - return _invoke_function("current_database") + fn = "tuple_sketch_agg_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) @_try_remote_functions -def current_schema() -> Column: - """Returns the current database. +def tuple_sketch_agg_integer( + key: "ColumnOrName", + summary: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch with integer summaries built from the key and summary columns. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 + + Parameters + ---------- + key : :class:`~pyspark.sql.Column` or column name + The column containing key values. + A column that evaluates to an array, binary, double, float, integer, long, or string. + summary : :class:`~pyspark.sql.Column` or column name + The column containing integer summary values. + A column that evaluates to an integer. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.current_catalog` - :meth:`pyspark.sql.functions.current_database` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` + :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` + :meth:`pyspark.sql.functions.tuple_union_agg_integer` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_schema()).show() - +----------------+ - |current_schema()| - +----------------+ - | default| - +----------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +----------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| + +----------------------------------------------------------------------------+ + | 2.0| + +----------------------------------------------------------------------------+ """ - return _invoke_function("current_schema") + fn = "tuple_sketch_agg_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, key, summary, _lgNomEntries, _mode) @_try_remote_functions -def current_user() -> Column: - """Returns the current database. +def tuple_union_agg_double( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the union of the double TupleSketch objects in the input column. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the merged TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.session_user` + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_double` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.current_user()).show() # doctest: +SKIP - +--------------+ - |current_user()| - +--------------+ - | ruifeng.zheng| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(3, 30.0), (4, 40.0)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_union_agg_double("sketch"))).show() + +---------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_union_agg_double(sketch, 12, sum))| + +---------------------------------------------------------------------+ + | 4.0| + +---------------------------------------------------------------------+ """ - return _invoke_function("current_user") + fn = "tuple_union_agg_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) @_try_remote_functions -def user() -> Column: - """Returns the current database. +def tuple_union_agg_integer( + col: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the union of the integer TupleSketch objects in the input column. - .. versionadded:: 3.5.0 + .. versionadded:: 4.2.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the merged TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.current_user` - :meth:`pyspark.sql.functions.session_user` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_integer` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.user()).show() # doctest: +SKIP - +--------------+ - | user()| - +--------------+ - | ruifeng.zheng| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(3, 30), (4, 40)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_union_agg_integer("sketch"))).show() + +-----------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_union_agg_integer(sketch, 12, sum))| + +-----------------------------------------------------------------------+ + | 4.0| + +-----------------------------------------------------------------------+ """ - return _invoke_function("user") + fn = "tuple_union_agg_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col, _lgNomEntries, _mode) @_try_remote_functions -def session_user() -> Column: - """Returns the user name of current execution context. +def tuple_intersection_agg_double( + col: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the intersection of the double TupleSketch objects in the input column. - .. versionadded:: 4.0.0 + .. versionadded:: 4.2.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the intersected TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.user` - :meth:`pyspark.sql.functions.current_user` + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_intersection_double` Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.session_user()).show() # doctest: +SKIP - +--------------+ - |session_user()| - +--------------+ - | ruifeng.zheng| - +--------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([(1, 10.0), (2, 20.0), (3, 30.0)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(2, 40.0), (3, 50.0), (4, 60.0)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_double("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_double(sf.tuple_intersection_agg_double("sketch"))).show() + +------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_intersection_agg_double(sketch, sum))| + +------------------------------------------------------------------------+ + | 2.0| + +------------------------------------------------------------------------+ """ - return _invoke_function("session_user") + fn = "tuple_intersection_agg_double" + if mode is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(mode)) @_try_remote_functions -def uuid(seed: Optional[Union[Column, int]] = None) -> Column: - """Returns an universally unique identifier (UUID) string. - The value is returned as a canonical UUID 36-character string. +def tuple_intersection_agg_integer( + col: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: + """ + Aggregate function: returns the compact binary representation of the Datasketches + TupleSketch that is the intersection of the integer TupleSketch objects in the input column. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - seed : :class:`~pyspark.sql.Column` or int - Optional random number seed to use. - - Examples - -------- - Example 1: Generate UUIDs with random seed + col : :class:`~pyspark.sql.Column` or column name + The column containing binary TupleSketch representations. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" - >>> from pyspark.sql import functions as sf - >>> spark.range(5).select(sf.uuid()).show(truncate=False) # doctest: +SKIP - +------------------------------------+ - |uuid() | - +------------------------------------+ - |627ae05e-b319-42b5-b4e4-71c8c9754dd1| - |f781cce5-a2e2-464d-bc8b-426ff448e404| - |15e2e66e-8416-4ea2-af3c-409363408189| - |fb1d6178-7676-4791-baa9-f2ddcc494515| - |d48665e8-2657-4c6b-b7c8-8ae0cd646e41| - +------------------------------------+ + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the intersected TupleSketch. - Example 2: Generate UUIDs with a specified seed + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_intersection_integer` + Examples + -------- >>> from pyspark.sql import functions as sf - >>> spark.range(0, 5, 1, 1).select(sf.uuid(seed=123)).show(truncate=False) - +------------------------------------+ - |uuid() | - +------------------------------------+ - |4c99192d-23d6-4d88-b814-a634398120f0| - |af506873-3c53-41e3-8354-a24856b8de8a| - |7b4b370e-e867-47e2-93c0-f6990463a12d| - |1c4d1733-ff1a-4a6c-b144-0b0345adf0d0| - |7478f235-f8bc-4112-8e59-a28f50e46890| - +------------------------------------+ + >>> df1 = spark.createDataFrame([(1, 10), (2, 20), (3, 30)], ["key", "value"]) + >>> df1 = df1.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df2 = spark.createDataFrame([(2, 40), (3, 50), (4, 60)], ["key", "value"]) + >>> df2 = df2.agg(sf.tuple_sketch_agg_integer("key", "value").alias("sketch")) + >>> df3 = df1.union(df2) + >>> df3.agg(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_agg_integer("sketch"))).show() + +--------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_intersection_agg_integer(sketch, sum))| + +--------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - if seed is None: - return _invoke_function("uuid") + fn = "tuple_intersection_agg_integer" + if mode is None: + return _invoke_function_over_columns(fn, col) else: - return _invoke_function("uuid", _to_java_column(lit(seed))) + return _invoke_function_over_columns(fn, col, lit(mode)) @_try_remote_functions -def assert_true(col: "ColumnOrName", errMsg: Optional[Union[Column, str]] = None) -> Column: +def kll_sketch_agg_bigint( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, +) -> Column: """ - Returns `null` if the input column is `true`; throws an exception - with the provided error message otherwise. - - .. versionadded:: 3.1.0 + Aggregate function: returns the compact binary representation of the Datasketches + KllLongsSketch built with the values in the input column. The optional k parameter + controls the size and accuracy of the sketch (default 200, range 8-65535). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - column name or column that represents the input column to test. - A column that evaluates to a boolean. - errMsg : :class:`~pyspark.sql.Column` or literal string, optional - A Python string literal or column containing the error message. - A column that evaluates to a string. + The column containing bigint values to aggregate. + A column that evaluates to an integral. + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (default 200, range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - `null` if the input column is `true` otherwise throws an error with specified message. - Returns a column that always evaluates to NULL. - - See Also - -------- - :meth:`pyspark.sql.functions.raise_error` + The binary representation of the KllLongsSketch. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([(0, 1)], ['a', 'b']) - >>> df.select('*', sf.assert_true(df.a < df.b)).show() - +---+---+--------------------------------------------+ - | a| b|assert_true((a < b), '(a < b)' is not true!)| - +---+---+--------------------------------------------+ - | 0| 1| NULL| - +---+---+--------------------------------------------+ - - >>> df.select('*', sf.assert_true(df.a < df.b, df.a)).show() - +---+---+-----------------------+ - | a| b|assert_true((a < b), a)| - +---+---+-----------------------+ - | 0| 1| NULL| - +---+---+-----------------------+ - - >>> df.select('*', sf.assert_true(df.a < df.b, 'error')).show() - +---+---+---------------------------+ - | a| b|assert_true((a < b), error)| - +---+---+---------------------------+ - | 0| 1| NULL| - +---+---+---------------------------+ - - >>> df.select('*', sf.assert_true(df.a > df.b, 'My error msg')).show() # doctest: +SKIP - ... - java.lang.RuntimeException: My error msg - ... + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> result = df.agg(sf.kll_sketch_agg_bigint("value")).first()[0] + >>> result is not None and len(result) > 0 + True """ - errMsg = _enum_to_value(errMsg) - if errMsg is None: - return _invoke_function_over_columns("assert_true", col) - if not isinstance(errMsg, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "errMsg", - "arg_type": type(errMsg).__name__, - }, - ) - return _invoke_function_over_columns("assert_true", col, lit(errMsg)) + fn = "kll_sketch_agg_bigint" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def raise_error(errMsg: Union[Column, str]) -> Column: +def kll_sketch_agg_float( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, +) -> Column: """ - Throws an exception with the provided error message. - - .. versionadded:: 3.1.0 + Aggregate function: returns the compact binary representation of the Datasketches + KllFloatsSketch built with the values in the input column. The optional k parameter + controls the size and accuracy of the sketch (default 200, range 8-65535). - .. versionchanged:: 3.4.0 - Supports Spark Connect. + .. versionadded:: 4.1.0 Parameters ---------- - errMsg : :class:`~pyspark.sql.Column` or literal string - A Python string literal or column containing the error message. - A column that evaluates to a string. + col : :class:`~pyspark.sql.Column` or column name + The column containing float values to aggregate + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (default 200, range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - throws an error with specified message. - Returns a column that always evaluates to NULL. - - See Also - -------- - :meth:`pyspark.sql.functions.assert_true` + The binary representation of the KllFloatsSketch. Examples -------- - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select(sf.raise_error("My error message")).show() # doctest: +SKIP - ... - java.lang.RuntimeException: My error message - ... + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> result = df.agg(sf.kll_sketch_agg_float("value")).first()[0] + >>> result is not None and len(result) > 0 + True """ - errMsg = _enum_to_value(errMsg) - if not isinstance(errMsg, (str, Column)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "errMsg", - "arg_type": type(errMsg).__name__, - }, - ) - return _invoke_function_over_columns("raise_error", lit(errMsg)) + fn = "kll_sketch_agg_float" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def hmac( - key: "ColumnOrName", - message: "ColumnOrName", - algorithm: Optional["ColumnOrName"] = None, +def kll_sketch_agg_double( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, ) -> Column: """ - Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the - given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with :func:`hex` or - :func:`base64` for a textual value. The default algorithm is 'SHA-256'. + Aggregate function: returns the compact binary representation of the Datasketches + KllDoublesSketch built with the values in the input column. The optional k parameter + controls the size and accuracy of the sketch (default 200, range 8-65535). - .. versionadded:: 4.3.0 + .. versionadded:: 4.1.0 Parameters ---------- - key : :class:`~pyspark.sql.Column` or column name - The secret key, as a binary value. - message : :class:`~pyspark.sql.Column` or column name - The message to authenticate, as a binary value. - algorithm : :class:`~pyspark.sql.Column` or column name, optional - The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. - The default is SHA-256. + col : :class:`~pyspark.sql.Column` or column name + The column containing double values to aggregate. + A column that evaluates to a float or double. + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (default 200, range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains the raw HMAC bytes. + The binary representation of the KllDoublesSketch. Examples -------- - - Example 1: Compute the HMAC with the default SHA-256 algorithm. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) - >>> df.select(sf.hex(sf.hmac(df.key, df.message))).show(truncate=False) - +----------------------------------------------------------------+ - |hex(hmac(key, message, SHA-256)) | - +----------------------------------------------------------------+ - |6E9EF29B75FFFC5B7ABAE527D58FDADB2FE42E7219011976917343065F58ED4A| - +----------------------------------------------------------------+ - - Example 2: Compute the HMAC with an explicit algorithm. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) - >>> df.select(sf.hex(sf.hmac(df.key, df.message, sf.lit("SHA-1")))).show(truncate=False) - +----------------------------------------+ - |hex(hmac(key, message, SHA-1)) | - +----------------------------------------+ - |2088DF74D5F2146B48146CAF4965377E9D0BE3A4| - +----------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> result = df.agg(sf.kll_sketch_agg_double("value")).first()[0] + >>> result is not None and len(result) > 0 + True """ - if algorithm is None: - return _invoke_function_over_columns("hmac", key, message) + fn = "kll_sketch_agg_double" + if k is None: + return _invoke_function_over_columns(fn, col) else: - return _invoke_function_over_columns("hmac", key, message, algorithm) + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def aes_encrypt( - input: "ColumnOrName", - key: "ColumnOrName", - mode: Optional["ColumnOrName"] = None, - padding: Optional["ColumnOrName"] = None, - iv: Optional["ColumnOrName"] = None, - aad: Optional["ColumnOrName"] = None, +def kll_merge_agg_bigint( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, ) -> Column: """ - Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. - Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, - `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization - vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 - bytes for GCM. If not provided, a random vector will be generated and prepended to the - output. Optional additional authenticated data (AAD) is only supported for GCM. If provided - for encryption, the identical AAD value must be provided for decryption. The default mode is - GCM. + Aggregate function: merges binary KllLongsSketch representations and returns the + merged sketch. The optional k parameter controls the size and accuracy of the merged + sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value + from the first input sketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.2 Parameters ---------- - input : :class:`~pyspark.sql.Column` or column name - The binary value to encrypt. - A column that evaluates to a binary. - key : :class:`~pyspark.sql.Column` or column name - The passphrase to use to encrypt the data. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or str, optional - Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - GCM, CBC. - A column that evaluates to a string. - padding : :class:`~pyspark.sql.Column` or column name, optional - Specifies how to pad messages whose length is not a multiple of the block size. Valid - values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - for CBC. - A column that evaluates to a string. - iv : :class:`~pyspark.sql.Column` or column name, optional - Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - "". 16-byte array for CBC mode. 12-byte array for GCM mode. - A column that evaluates to a binary. - aad : :class:`~pyspark.sql.Column` or column name, optional - Optional additional authenticated data. Only supported for GCM mode. This can be any - free-form input and must be provided for both encryption and decryption. - A column that evaluates to a binary. + col : :class:`~pyspark.sql.Column` or column name + The column containing binary KllLongsSketch representations + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains an encrypted value. - Returns a column that evaluates to a binary. - - See Also - -------- - :meth:`pyspark.sql.functions.aes_decrypt` - :meth:`pyspark.sql.functions.try_aes_decrypt` + The merged binary representation of the KllLongsSketch. Examples -------- - - Example 1: Encrypt data with key, mode, padding, iv and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "000000000000000000000000", "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "iv", "aad"] - ... ) - >>> df.select(sf.base64(sf.aes_encrypt( - ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex")), df.aad) - ... )).show(truncate=False) - +-----------------------------------------------------------------------+ - |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), aad))| - +-----------------------------------------------------------------------+ - |AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4 | - +-----------------------------------------------------------------------+ - - Example 2: Encrypt data with key, mode, padding and iv. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "000000000000000000000000", "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "iv", "aad"] - ... ) - >>> df.select(sf.base64(sf.aes_encrypt( - ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex"))) - ... )).show(truncate=False) - +--------------------------------------------------------------------+ - |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), ))| - +--------------------------------------------------------------------+ - |AAAAAAAAAAAAAAAAQiYi+sRNYDAOTjdSEcYBFsAWPL1f | - +--------------------------------------------------------------------+ - - Example 3: Encrypt data with key, mode and padding. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark SQL", "1234567890abcdef", "ECB", "PKCS",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode", df.padding), - ... df.key, df.mode, df.padding - ... ).cast("STRING")).show(truncate=False) - +---------------------------------------------------------------------------------------------+ - |CAST(aes_decrypt(aes_encrypt(input, key, mode, padding, , ), key, mode, padding, ) AS STRING)| - +---------------------------------------------------------------------------------------------+ - |Spark SQL | - +---------------------------------------------------------------------------------------------+ - - Example 4: Encrypt data with key and mode. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark SQL", "0000111122223333", "ECB",)], - ... ["input", "key", "mode"] - ... ) - >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode"), - ... df.key, df.mode - ... ).cast("STRING")).show(truncate=False) - +---------------------------------------------------------------------------------------------+ - |CAST(aes_decrypt(aes_encrypt(input, key, mode, DEFAULT, , ), key, mode, DEFAULT, ) AS STRING)| - +---------------------------------------------------------------------------------------------+ - |Spark SQL | - +---------------------------------------------------------------------------------------------+ - - Example 5: Encrypt data with key. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "Spark SQL", "abcdefghijklmnop",)], - ... ["input", "key"] - ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(sf.base64(sf.aes_encrypt(df.input, df.key))), df.key - ... ).cast("STRING")).show(truncate=False) - +-------------------------------------------------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(base64(aes_encrypt(input, key, GCM, DEFAULT, , ))), key, GCM, DEFAULT, ) AS STRING)| - +-------------------------------------------------------------------------------------------------------------+ - |Spark SQL | - +-------------------------------------------------------------------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1,2,3], "INT") + >>> df2 = spark.createDataFrame([4,5,6], "INT") + >>> sketch1 = df1.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch2 = df2.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_bigint("sketch").alias("merged")) + >>> n = merged.select(sf.kll_sketch_get_n_bigint("merged")).first()[0] + >>> n + 6 """ - _mode = lit("GCM") if mode is None else mode - _padding = lit("DEFAULT") if padding is None else padding - _iv = lit("") if iv is None else iv - _aad = lit("") if aad is None else aad - return _invoke_function_over_columns("aes_encrypt", input, key, _mode, _padding, _iv, _aad) + fn = "kll_merge_agg_bigint" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def aes_decrypt( - input: "ColumnOrName", - key: "ColumnOrName", - mode: Optional["ColumnOrName"] = None, - padding: Optional["ColumnOrName"] = None, - aad: Optional["ColumnOrName"] = None, +def kll_merge_agg_float( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, ) -> Column: """ - Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, - 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', - 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is - only supported for GCM. If provided for encryption, the identical AAD value must be provided - for decryption. The default mode is GCM. + Aggregate function: merges binary KllFloatsSketch representations and returns the + merged sketch. The optional k parameter controls the size and accuracy of the merged + sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value + from the first input sketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.2 Parameters ---------- - input : :class:`~pyspark.sql.Column` or column name - The binary value to decrypt. - A column that evaluates to a binary. - key : :class:`~pyspark.sql.Column` or column name - The passphrase to use to decrypt the data. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or column name, optional - Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - GCM, CBC. - A column that evaluates to a string. - padding : :class:`~pyspark.sql.Column` or column name, optional - Specifies how to pad messages whose length is not a multiple of the block size. Valid - values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - for CBC. - A column that evaluates to a string. - aad : :class:`~pyspark.sql.Column` or column name, optional - Optional additional authenticated data. Only supported for GCM mode. This can be any - free-form input and must be provided for both encryption and decryption. - A column that evaluates to a binary. + col : :class:`~pyspark.sql.Column` or column name + The column containing binary KllFloatsSketch representations + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a decrypted value. - Returns a column that evaluates to a binary. - - See Also - -------- - :meth:`pyspark.sql.functions.aes_encrypt` - :meth:`pyspark.sql.functions.try_aes_decrypt` + The merged binary representation of the KllFloatsSketch. Examples -------- - - Example 1: Decrypt data with key, mode, padding and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", - ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "aad"] - ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad - ... ).cast("STRING")).show(truncate=False) - +---------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| - +---------------------------------------------------------------------+ - |Spark | - +---------------------------------------------------------------------+ - - Example 2: Decrypt data with key, mode and padding. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding - ... ).cast("STRING")).show(truncate=False) - +------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| - +------------------------------------------------------------------+ - |Spark | - +------------------------------------------------------------------+ - - Example 3: Decrypt data with key and mode. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode" - ... ).cast("STRING")).show(truncate=False) - +------------------------------------------------------------------+ - |CAST(aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| - +------------------------------------------------------------------+ - |Spark | - +------------------------------------------------------------------+ - - Example 4: Decrypt data with key. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", - ... "0000111122223333",)], - ... ["input", "key"] - ... ) - >>> df.select(sf.aes_decrypt( - ... sf.unhex(df.input), df.key - ... ).cast("STRING")).show(truncate=False) - +--------------------------------------------------------------+ - |CAST(aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| - +--------------------------------------------------------------+ - |Spark | - +--------------------------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "FLOAT") + >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "FLOAT") + >>> sketch1 = df1.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch2 = df2.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_float("sketch").alias("merged")) + >>> n = merged.select(sf.kll_sketch_get_n_float("merged")).first()[0] + >>> n + 6 """ - _mode = lit("GCM") if mode is None else mode - _padding = lit("DEFAULT") if padding is None else padding - _aad = lit("") if aad is None else aad - return _invoke_function_over_columns("aes_decrypt", input, key, _mode, _padding, _aad) + fn = "kll_merge_agg_float" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) @_try_remote_functions -def try_aes_decrypt( - input: "ColumnOrName", - key: "ColumnOrName", - mode: Optional["ColumnOrName"] = None, - padding: Optional["ColumnOrName"] = None, - aad: Optional["ColumnOrName"] = None, +def kll_merge_agg_double( + col: "ColumnOrName", + k: Optional[Union[int, Column]] = None, ) -> Column: """ - This is a special version of `aes_decrypt` that performs the same operation, - but returns a NULL value instead of raising an error if the decryption cannot be performed. - Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, - 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', - 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is - only supported for GCM. If provided for encryption, the identical AAD value must be provided - for decryption. The default mode is GCM. + Aggregate function: merges binary KllDoublesSketch representations and returns the + merged sketch. The optional k parameter controls the size and accuracy of the merged + sketch (range 8-65535). If k is not specified, the merged sketch adopts the k value + from the first input sketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.2 Parameters ---------- - input : :class:`~pyspark.sql.Column` or column name - The binary value to decrypt. - A column that evaluates to a binary. - key : :class:`~pyspark.sql.Column` or column name - The passphrase to use to decrypt the data. - A column that evaluates to a binary. - mode : :class:`~pyspark.sql.Column` or column name, optional - Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - GCM, CBC. - A column that evaluates to a string. - padding : :class:`~pyspark.sql.Column` or column name, optional - Specifies how to pad messages whose length is not a multiple of the block size. Valid - values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - for CBC. - A column that evaluates to a string. - aad : :class:`~pyspark.sql.Column` or column name, optional - Optional additional authenticated data. Only supported for GCM mode. This can be any - free-form input and must be provided for both encryption and decryption. - A column that evaluates to a binary. + col : :class:`~pyspark.sql.Column` or column name + The column containing binary KllDoublesSketch representations + k : :class:`~pyspark.sql.Column` or int, optional + The k parameter that controls size and accuracy (range 8-65535) + A column that evaluates to an integer. Must be a constant. Returns ------- :class:`~pyspark.sql.Column` - A new column that contains a decrypted value or a NULL value. - Returns a column that evaluates to a binary. - - See Also - -------- - :meth:`pyspark.sql.functions.aes_encrypt` - :meth:`pyspark.sql.functions.aes_decrypt` + The merged binary representation of the KllDoublesSketch. Examples -------- + >>> from pyspark.sql import functions as sf + >>> df1 = spark.createDataFrame([1.0,2.0,3.0], "DOUBLE") + >>> df2 = spark.createDataFrame([4.0,5.0,6.0], "DOUBLE") + >>> sketch1 = df1.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch2 = df2.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> merged = sketch1.union(sketch2).agg(sf.kll_merge_agg_double("sketch").alias("merged")) + >>> n = merged.select(sf.kll_sketch_get_n_double("merged")).first()[0] + >>> n + 6 + """ + fn = "kll_merge_agg_double" + if k is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(k)) - Example 1: Decrypt data with key, mode, padding and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", - ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", - ... "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "aad"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad - ... ).cast("STRING")).show(truncate=False) - +-------------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| - +-------------------------------------------------------------------------+ - |Spark | - +-------------------------------------------------------------------------+ - - Example 2: Failed to decrypt data with key, mode, padding and aad. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT", - ... "This is an AAD mixed into the input",)], - ... ["input", "key", "mode", "padding", "aad"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad - ... ).cast("STRING")).show(truncate=False) - +-------------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| - +-------------------------------------------------------------------------+ - |NULL | - +-------------------------------------------------------------------------+ - - Example 3: Decrypt data with key, mode and padding. - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode", df.padding - ... ).cast("STRING")).show(truncate=False) - +----------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| - +----------------------------------------------------------------------+ - |Spark | - +----------------------------------------------------------------------+ - - Example 4: Decrypt data with key and mode. - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", - ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], - ... ["input", "key", "mode", "padding"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unbase64(df.input), df.key, "mode" - ... ).cast("STRING")).show(truncate=False) - +----------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| - +----------------------------------------------------------------------+ - |Spark | - +----------------------------------------------------------------------+ +@_try_remote_functions +def kll_sketch_to_string_bigint(col: "ColumnOrName") -> Column: + """ + Returns a string with human readable summary information about the KLL bigint sketch. - Example 5: Decrypt data with key. + .. versionadded:: 4.1.0 - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([( - ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", - ... "0000111122223333",)], - ... ["input", "key"] - ... ) - >>> df.select(sf.try_aes_decrypt( - ... sf.unhex(df.input), df.key - ... ).cast("STRING")).show(truncate=False) - +------------------------------------------------------------------+ - |CAST(try_aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| - +------------------------------------------------------------------+ - |Spark | - +------------------------------------------------------------------+ + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The KLL bigint sketch binary representation. + A column that evaluates to a binary. + + Returns + ------- + :class:`~pyspark.sql.Column` + A string representation of the sketch. + Returns a column that evaluates to a string. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_to_string_bigint("sketch")).first()[0] + >>> "kll" in result.lower() + True """ - _mode = lit("GCM") if mode is None else mode - _padding = lit("DEFAULT") if padding is None else padding - _aad = lit("") if aad is None else aad - return _invoke_function_over_columns("try_aes_decrypt", input, key, _mode, _padding, _aad) + fn = "kll_sketch_to_string_bigint" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def input_file_block_length() -> Column: +def kll_sketch_to_string_float(col: "ColumnOrName") -> Column: """ - Returns the length of the block being read, or -1 if not available. + Returns a string with human readable summary information about the KLL float sketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 - See Also - -------- - :meth:`pyspark.sql.functions.input_file_name` - :meth:`pyspark.sql.functions.input_file_block_start` + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The KLL float sketch binary representation. + A column that evaluates to a binary. + + Returns + ------- + :class:`~pyspark.sql.Column` + A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") - >>> df.select(sf.input_file_block_length()).show() - +-------------------------+ - |input_file_block_length()| - +-------------------------+ - | 87| - | 87| - | 87| - | 87| - | 87| - | 87| - | 87| - | 87| - +-------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_to_string_float("sketch")).first()[0] + >>> "kll" in result.lower() + True """ - return _invoke_function_over_columns("input_file_block_length") + fn = "kll_sketch_to_string_float" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def input_file_block_start() -> Column: +def kll_sketch_to_string_double(col: "ColumnOrName") -> Column: """ - Returns the start offset of the block being read, or -1 if not available. + Returns a string with human readable summary information about the KLL double sketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 - See Also - -------- - :meth:`pyspark.sql.functions.input_file_name` - :meth:`pyspark.sql.functions.input_file_block_length` + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + The KLL double sketch binary representation. + A column that evaluates to a binary. + + Returns + ------- + :class:`~pyspark.sql.Column` + A string representation of the sketch. + Returns a column that evaluates to a string. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") - >>> df.select(sf.input_file_block_start()).show() - +------------------------+ - |input_file_block_start()| - +------------------------+ - | 0| - | 0| - | 0| - | 0| - | 0| - | 0| - | 0| - | 0| - +------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_to_string_double("sketch")).first()[0] + >>> "kll" in result.lower() + True """ - return _invoke_function_over_columns("input_file_block_start") + fn = "kll_sketch_to_string_double" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def reflect(*cols: "ColumnOrName") -> Column: +def kll_sketch_get_n_bigint(col: "ColumnOrName") -> Column: """ - Calls a method with reflection. + Returns the number of items collected in the KLL bigint sketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a Column representing literal string for the class name, - and the second element should be a Column representing literal string for the method name, - and the remaining are input arguments (Columns or column names) to the Java method. + col : :class:`~pyspark.sql.Column` or column name + The KLL bigint sketch binary representation. + A column that evaluates to a binary. - See Also - -------- - :meth:`pyspark.sql.functions.java_method` - :meth:`pyspark.sql.functions.try_reflect` + Returns + ------- + :class:`~pyspark.sql.Column` + The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) - >>> df.select( - ... sf.reflect(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') - ... ).show(truncate=False) - +--------------------------------------+ - |reflect(java.util.UUID, fromString, a)| - +--------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +--------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_n_bigint("sketch")).show() + +-------------------------------+ + |kll_sketch_get_n_bigint(sketch)| + +-------------------------------+ + | 5| + +-------------------------------+ """ - return _invoke_function_over_seq_of_columns("reflect", cols) + fn = "kll_sketch_get_n_bigint" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def java_method(*cols: "ColumnOrName") -> Column: +def kll_sketch_get_n_float(col: "ColumnOrName") -> Column: """ - Calls a method with reflection. + Returns the number of items collected in the KLL float sketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a Column representing literal string for the class name, - and the second element should be a Column representing literal string for the method name, - and the remaining are input arguments (Columns or column names) to the Java method. + col : :class:`~pyspark.sql.Column` or column name + The KLL float sketch binary representation. + A column that evaluates to a binary. - See Also - -------- - :meth:`pyspark.sql.functions.reflect` - :meth:`pyspark.sql.functions.try_reflect` + Returns + ------- + :class:`~pyspark.sql.Column` + The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- - Example 1: Reflecting a method call with a column argument - - >>> import pyspark.sql.functions as sf - >>> spark.range(1).select( - ... sf.java_method( - ... sf.lit("java.util.UUID"), - ... sf.lit("fromString"), - ... sf.lit("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2") - ... ) - ... ).show(truncate=False) - +-----------------------------------------------------------------------------+ - |java_method(java.util.UUID, fromString, a5cf6c42-0c85-418f-af6c-3e4e5b1328f2)| - +-----------------------------------------------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +-----------------------------------------------------------------------------+ - - Example 2: Reflecting a method call with a column name argument - - >>> import pyspark.sql.functions as sf - >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) - >>> df.select( - ... sf.java_method(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') - ... ).show(truncate=False) - +------------------------------------------+ - |java_method(java.util.UUID, fromString, a)| - +------------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +------------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_n_float("sketch")).show() + +------------------------------+ + |kll_sketch_get_n_float(sketch)| + +------------------------------+ + | 5| + +------------------------------+ """ - return _invoke_function_over_seq_of_columns("java_method", cols) + fn = "kll_sketch_get_n_float" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def try_reflect(*cols: "ColumnOrName") -> Column: +def kll_sketch_get_n_double(col: "ColumnOrName") -> Column: """ - This is a special version of `reflect` that performs the same operation, but returns a NULL - value instead of raising an error if the invoke method thrown exception. - + Returns the number of items collected in the KLL double sketch. - .. versionadded:: 4.0.0 + .. versionadded:: 4.1.0 Parameters ---------- - cols : :class:`~pyspark.sql.Column` or column name - the first element should be a Column representing literal string for the class name, - and the second element should be a Column representing literal string for the method name, - and the remaining are input arguments (Columns or column names) to the Java method. + col : :class:`~pyspark.sql.Column` or column name + The KLL double sketch binary representation. + A column that evaluates to a binary. - See Also - -------- - :meth:`pyspark.sql.functions.reflect` - :meth:`pyspark.sql.functions.java_method` + Returns + ------- + :class:`~pyspark.sql.Column` + The count of items in the sketch. + Returns a column that evaluates to a long. Examples -------- - Example 1: Reflecting a method call with arguments - - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2",)], ["a"]) - >>> df.select( - ... sf.try_reflect(sf.lit("java.util.UUID"), sf.lit("fromString"), "a") - ... ).show(truncate=False) - +------------------------------------------+ - |try_reflect(java.util.UUID, fromString, a)| - +------------------------------------------+ - |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | - +------------------------------------------+ - - Example 2: Exception in the reflection call, resulting in null - >>> from pyspark.sql import functions as sf - >>> spark.range(1).select( - ... sf.try_reflect(sf.lit("scala.Predef"), sf.lit("require"), sf.lit(False)) - ... ).show(truncate=False) - +-----------------------------------------+ - |try_reflect(scala.Predef, require, false)| - +-----------------------------------------+ - |NULL | - +-----------------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_n_double("sketch")).show() + +-------------------------------+ + |kll_sketch_get_n_double(sketch)| + +-------------------------------+ + | 5| + +-------------------------------+ """ - return _invoke_function_over_seq_of_columns("try_reflect", cols) + fn = "kll_sketch_get_n_double" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def version() -> Column: +def kll_sketch_merge_bigint(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns the Spark version. The string contains 2 fields, the first being a release version - and the second being a git revision. + Merges two KLL bigint sketch buffers together into one. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 + + Parameters + ---------- + left : :class:`~pyspark.sql.Column` or column name + The first KLL bigint sketch. + A column that evaluates to a binary. + right : :class:`~pyspark.sql.Column` or column name + The second KLL bigint sketch. + A column that evaluates to a binary. + + Returns + ------- + :class:`~pyspark.sql.Column` + The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> spark.range(1).select(sf.version()).show(truncate=False) # doctest: +SKIP - +----------------------------------------------+ - |version() | - +----------------------------------------------+ - |4.0.0 4f8d1f575e99aeef8990c63a9614af0fc5479330| - +----------------------------------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_merge_bigint("sketch", "sketch")).first()[0] + >>> result is not None and len(result) > 0 + True """ - return _invoke_function_over_columns("version") + fn = "kll_sketch_merge_bigint" + return _invoke_function_over_columns(fn, left, right) @_try_remote_functions -def typeof(col: "ColumnOrName") -> Column: +def kll_sketch_merge_float(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Return DDL-formatted type string for the data type of the input. + Merges two KLL float sketch buffers together into one. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + left : :class:`~pyspark.sql.Column` or column name + The first KLL float sketch. + A column that evaluates to a binary. + right : :class:`~pyspark.sql.Column` or column name + The second KLL float sketch. + A column that evaluates to a binary. + + Returns + ------- + :class:`~pyspark.sql.Column` + The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(True, 1, 1.0, 'xyz',)], ['a', 'b', 'c', 'd']) - >>> df.select(sf.typeof(df.a), sf.typeof(df.b), sf.typeof('c'), sf.typeof('d')).show() - +---------+---------+---------+---------+ - |typeof(a)|typeof(b)|typeof(c)|typeof(d)| - +---------+---------+---------+---------+ - | boolean| bigint| double| string| - +---------+---------+---------+---------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_merge_float("sketch", "sketch")).first()[0] + >>> result is not None and len(result) > 0 + True """ - return _invoke_function_over_columns("typeof", col) + fn = "kll_sketch_merge_float" + return _invoke_function_over_columns(fn, left, right) @_try_remote_functions -def bitmap_bit_position(col: "ColumnOrName") -> Column: +def kll_sketch_merge_double(left: "ColumnOrName", right: "ColumnOrName") -> Column: """ - Returns the bit position for the given input column. + Merges two KLL double sketch buffers together into one. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column. - A column that evaluates to a long. + left : :class:`~pyspark.sql.Column` or column name + The first KLL double sketch. + A column that evaluates to a binary. + right : :class:`~pyspark.sql.Column` or column name + The second KLL double sketch. + A column that evaluates to a binary. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` + Returns + ------- + :class:`~pyspark.sql.Column` + The merged KLL sketch. + Returns a column that evaluates to a binary. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(123,)], ['a']) - >>> df.select('*', sf.bitmap_bit_position('a')).show() - +---+----------------------+ - | a|bitmap_bit_position(a)| - +---+----------------------+ - |123| 122| - +---+----------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> result = sketch_df.select(sf.kll_sketch_merge_double("sketch", "sketch")).first()[0] + >>> result is not None and len(result) > 0 + True """ - return _invoke_function_over_columns("bitmap_bit_position", col) + fn = "kll_sketch_merge_double" + return _invoke_function_over_columns(fn, left, right) @_try_remote_functions -def bitmap_bucket_number(col: "ColumnOrName") -> Column: +def kll_sketch_get_quantile_bigint(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: """ - Returns the bucket number for the given input column. + Extracts a quantile value from a KLL bigint sketch given an input rank value. + The rank can be a single value or an array. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input column. - A column that evaluates to a long. + sketch : :class:`~pyspark.sql.Column` or column name + The KLL bigint sketch binary representation. + A column that evaluates to a binary. + rank : :class:`~pyspark.sql.Column` or column name + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_count` - :meth:`pyspark.sql.functions.bitmap_or_agg` + Returns + ------- + :class:`~pyspark.sql.Column` + The quantile value(s). + Returns a column that evaluates to a long, or an array of longs if the rank + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(123,)], ['a']) - >>> df.select('*', sf.bitmap_bucket_number('a')).show() - +---+-----------------------+ - | a|bitmap_bucket_number(a)| - +---+-----------------------+ - |123| 1| - +---+-----------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_quantile_bigint("sketch", sf.lit(0.5))).show() + +-------------------------------------------+ + |kll_sketch_get_quantile_bigint(sketch, 0.5)| + +-------------------------------------------+ + | 3| + +-------------------------------------------+ """ - return _invoke_function_over_columns("bitmap_bucket_number", col) + fn = "kll_sketch_get_quantile_bigint" + return _invoke_function_over_columns(fn, sketch, rank) @_try_remote_functions -def bitmap_count(col: "ColumnOrName") -> Column: +def kll_sketch_get_quantile_float(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: """ - Returns the number of set bits in the input bitmap. + Extracts a quantile value from a KLL float sketch given an input rank value. + The rank can be a single value or an array. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The input bitmap. + sketch : :class:`~pyspark.sql.Column` or column name + The KLL float sketch binary representation. + A column that evaluates to a binary. + rank : :class:`~pyspark.sql.Column` or column name + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_bit_position` - :meth:`pyspark.sql.functions.bitmap_bucket_number` - :meth:`pyspark.sql.functions.bitmap_construct_agg` - :meth:`pyspark.sql.functions.bitmap_or_agg` + Returns + ------- + :class:`~pyspark.sql.Column` + The quantile value(s). + Returns a column that evaluates to a float, or an array of floats if the rank + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("FFFF",)], ["a"]) - >>> df.select(sf.bitmap_count(sf.to_binary(df.a, sf.lit("hex")))).show() - +-------------------------------+ - |bitmap_count(to_binary(a, hex))| - +-------------------------------+ - | 16| - +-------------------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_quantile_float("sketch", sf.lit(0.5))).show() + +------------------------------------------+ + |kll_sketch_get_quantile_float(sketch, 0.5)| + +------------------------------------------+ + | 3.0| + +------------------------------------------+ """ - return _invoke_function_over_columns("bitmap_count", col) + fn = "kll_sketch_get_quantile_float" + return _invoke_function_over_columns(fn, sketch, rank) @_try_remote_functions -def bitmap_and(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def kll_sketch_get_quantile_double(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: """ - Returns a bitmap that is the bitwise AND of two input bitmaps. + Extracts a quantile value from a KLL double sketch given an input rank value. + The rank can be a single value or an array. - .. versionadded:: 4.4.0 + .. versionadded:: 4.1.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_andnot` - :meth:`pyspark.sql.functions.bitmap_or` - :meth:`pyspark.sql.functions.bitmap_xor` + sketch : :class:`~pyspark.sql.Column` or column name + The KLL double sketch binary representation. + A column that evaluates to a binary. + rank : :class:`~pyspark.sql.Column` or column name + The rank value(s) to extract (between 0.0 and 1.0). + A column that evaluates to a double or array. Must be a constant. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + The quantile value(s). + Returns a column that evaluates to a double, or an array of doubles if the rank + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) - >>> df.select(sf.bitmap_and( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[70 00 00 00 00 0...| - +--------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_quantile_double("sketch", sf.lit(0.5))).show() + +-------------------------------------------+ + |kll_sketch_get_quantile_double(sketch, 0.5)| + +-------------------------------------------+ + | 3.0| + +-------------------------------------------+ """ - return _invoke_function_over_columns("bitmap_and", left, right) + fn = "kll_sketch_get_quantile_double" + return _invoke_function_over_columns(fn, sketch, rank) @_try_remote_functions -def bitmap_or(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def kll_sketch_get_rank_bigint(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: """ - Returns a bitmap that is the bitwise OR of two input bitmaps. + Extracts a rank value from a KLL bigint sketch given an input quantile value. + The quantile can be a single value or an array. - .. versionadded:: 4.4.0 + .. versionadded:: 4.1.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_and` - :meth:`pyspark.sql.functions.bitmap_andnot` - :meth:`pyspark.sql.functions.bitmap_xor` + sketch : :class:`~pyspark.sql.Column` or column name + The KLL bigint sketch binary representation. + A column that evaluates to a binary. + quantile : :class:`~pyspark.sql.Column` or column name + The quantile value(s) to lookup. + A column that evaluates to a long or array. Must be a constant. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double, or an array of doubles if the quantile + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("10", "20")], ["left", "right"]) - >>> df.select(sf.bitmap_or( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[30 00 00 00 00 0...| - +--------------------+ + >>> df = spark.createDataFrame([1,2,3,4,5], "INT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_rank_bigint("sketch", sf.lit(3))).show() + +-------------------------------------+ + |kll_sketch_get_rank_bigint(sketch, 3)| + +-------------------------------------+ + | 0.6| + +-------------------------------------+ """ - return _invoke_function_over_columns("bitmap_or", left, right) + fn = "kll_sketch_get_rank_bigint" + return _invoke_function_over_columns(fn, sketch, quantile) @_try_remote_functions -def bitmap_andnot(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def kll_sketch_get_rank_float(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: """ - Returns a bitmap that is the bitwise AND NOT of two input bitmaps. + Extracts a rank value from a KLL float sketch given an input quantile value. + The quantile can be a single value or an array. - .. versionadded:: 4.4.0 + .. versionadded:: 4.1.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_and` - :meth:`pyspark.sql.functions.bitmap_or` - :meth:`pyspark.sql.functions.bitmap_xor` + sketch : :class:`~pyspark.sql.Column` or column name + The KLL float sketch binary representation. + A column that evaluates to a binary. + quantile : :class:`~pyspark.sql.Column` or column name + The quantile value(s) to lookup. + A column that evaluates to a float or array. Must be a constant. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double, or an array of doubles if the quantile + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) - >>> df.select(sf.bitmap_andnot( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[80 00 00 00 00 0...| - +--------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") + >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_rank_float("sketch", sf.lit(3.0))).show() + +--------------------------------------+ + |kll_sketch_get_rank_float(sketch, 3.0)| + +--------------------------------------+ + | 0.6| + +--------------------------------------+ """ - return _invoke_function_over_columns("bitmap_andnot", left, right) + fn = "kll_sketch_get_rank_float" + return _invoke_function_over_columns(fn, sketch, quantile) @_try_remote_functions -def bitmap_xor(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def kll_sketch_get_rank_double(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: """ - Returns a bitmap that is the bitwise XOR of two input bitmaps. + Extracts a rank value from a KLL double sketch given an input quantile value. + The quantile can be a single value or an array. - .. versionadded:: 4.4.0 + .. versionadded:: 4.1.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The left input bitmap. - right : :class:`~pyspark.sql.Column` or column name - The right input bitmap. - - See Also - -------- - :meth:`pyspark.sql.functions.bitmap_and` - :meth:`pyspark.sql.functions.bitmap_andnot` - :meth:`pyspark.sql.functions.bitmap_or` + sketch : :class:`~pyspark.sql.Column` or column name + The KLL double sketch binary representation. + A column that evaluates to a binary. + quantile : :class:`~pyspark.sql.Column` or column name + The quantile value(s) to lookup. + A column that evaluates to a double or array. Must be a constant. - Notes - ----- - Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each - input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a - 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise - ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were - constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must - represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across - rows. + Returns + ------- + :class:`~pyspark.sql.Column` + The rank value(s) (between 0.0 and 1.0). + Returns a column that evaluates to a double, or an array of doubles if the quantile + argument is an array. Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) - >>> df.select(sf.bitmap_xor( - ... sf.to_binary("left", sf.lit("hex")), - ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() - +--------------------+ - | bitmap| - +--------------------+ - |[80 00 00 00 00 0...| - +--------------------+ + >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") + >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) + >>> sketch_df.select(sf.kll_sketch_get_rank_double("sketch", sf.lit(3.0))).show() + +---------------------------------------+ + |kll_sketch_get_rank_double(sketch, 3.0)| + +---------------------------------------+ + | 0.6| + +---------------------------------------+ """ - return _invoke_function_over_columns("bitmap_xor", left, right) - - -# ---------------------- Datasketch Functions ---------------------- + fn = "kll_sketch_get_rank_double" + return _invoke_function_over_columns(fn, sketch, quantile) @_try_remote_functions -def hll_sketch_estimate(col: "ColumnOrName") -> Column: +def theta_sketch_estimate(col: "ColumnOrName") -> Column: """ Returns the estimated number of unique values given the binary representation - of a Datasketches HllSketch. + of a Datasketches ThetaSketch. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- @@ -31425,2310 +30879,2593 @@ def hll_sketch_estimate(col: "ColumnOrName") -> Column: Returns ------- :class:`~pyspark.sql.Column` - The estimated number of unique values for the HllSketch. + The estimated number of unique values for the ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.hll_union` - :meth:`pyspark.sql.functions.hll_union_agg` - :meth:`pyspark.sql.functions.hll_sketch_agg` + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.theta_difference` + :meth:`pyspark.sql.functions.theta_union_agg` + :meth:`pyspark.sql.functions.theta_intersection_agg` + :meth:`pyspark.sql.functions.theta_sketch_agg` Examples -------- >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.hll_sketch_estimate(sf.hll_sketch_agg("value"))).show() - +----------------------------------------------+ - |hll_sketch_estimate(hll_sketch_agg(value, 12))| - +----------------------------------------------+ - | 3| - +----------------------------------------------+ + >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() + +--------------------------------------------------+ + |theta_sketch_estimate(theta_sketch_agg(value, 12))| + +--------------------------------------------------+ + | 3| + +--------------------------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - return _invoke_function("hll_sketch_estimate", _to_java_column(col)) + fn = "theta_sketch_estimate" + return _invoke_function_over_columns(fn, col) @_try_remote_functions -def hll_union( - col1: "ColumnOrName", col2: "ColumnOrName", allowDifferentLgConfigK: Optional[bool] = None +def theta_union( + col1: "ColumnOrName", col2: "ColumnOrName", lgNomEntries: Optional[Union[int, Column]] = None ) -> Column: """ - Merges two binary representations of Datasketches HllSketch objects, using a - Datasketches Union object. Throws an exception if sketches have different - lgConfigK values and allowDifferentLgConfigK is unset or set to false. + Merges two binary representations of Datasketches ThetaSketch objects, using a + Datasketches Union object. - .. versionadded:: 3.5.0 + .. versionadded:: 4.1.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name col2 : :class:`~pyspark.sql.Column` or column name - allowDifferentLgConfigK : bool, optional - Allow sketches with different lgConfigK values to be merged (defaults to false). + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries for the union operation + (must be between 4 and 26, defaults to 12) Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged HllSketch. + The binary representation of the merged ThetaSketch. See Also -------- - :meth:`pyspark.sql.functions.hll_union_agg` - :meth:`pyspark.sql.functions.hll_sketch_agg` - :meth:`pyspark.sql.functions.hll_sketch_estimate` + :meth:`pyspark.sql.functions.theta_union_agg` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- >>> from pyspark.sql import functions as sf >>> df = spark.createDataFrame([(1,4),(2,5),(2,5),(3,6)], "struct") >>> df = df.agg( - ... sf.hll_sketch_agg("v1").alias("sketch1"), - ... sf.hll_sketch_agg("v2").alias("sketch2") + ... sf.theta_sketch_agg("v1").alias("sketch1"), + ... sf.theta_sketch_agg("v2").alias("sketch2") ... ) - >>> df.select(sf.hll_sketch_estimate(sf.hll_union(df.sketch1, "sketch2"))).show() - +-------------------------------------------------------+ - |hll_sketch_estimate(hll_union(sketch1, sketch2, false))| - +-------------------------------------------------------+ - | 6| - +-------------------------------------------------------+ + >>> df.select(sf.theta_sketch_estimate(sf.theta_union(df.sketch1, "sketch2"))).show() + +--------------------------------------------------------+ + |theta_sketch_estimate(theta_union(sketch1, sketch2, 12))| + +--------------------------------------------------------+ + | 6| + +--------------------------------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - if allowDifferentLgConfigK is not None: - return _invoke_function( - "hll_union", - _to_java_column(col1), - _to_java_column(col2), - _enum_to_value(allowDifferentLgConfigK), + fn = "theta_union" + if lgNomEntries is not None: + return _invoke_function_over_columns( + fn, + col1, + col2, + lit(lgNomEntries), ) else: - return _invoke_function("hll_union", _to_java_column(col1), _to_java_column(col2)) + return _invoke_function_over_columns(fn, col1, col2) @_try_remote_functions -def kll_sketch_to_string_bigint(col: "ColumnOrName") -> Column: +def theta_intersection(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns a string with human readable summary information about the KLL bigint sketch. + Returns the intersection of two binary representations of Datasketches ThetaSketch + objects, using a Datasketches Intersection object. .. versionadded:: 4.1.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. - A column that evaluates to a binary. + col1 : :class:`~pyspark.sql.Column` or column name + col2 : :class:`~pyspark.sql.Column` or column name Returns ------- :class:`~pyspark.sql.Column` - A string representation of the sketch. - Returns a column that evaluates to a string. + The binary representation of the intersected ThetaSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.theta_intersection_agg` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_to_string_bigint("sketch")).first()[0] - >>> "kll" in result.lower() - True + >>> df = spark.createDataFrame([(1,1),(2,2),(3,2),(3,3)], "struct") + >>> df = df.agg( + ... sf.theta_sketch_agg("v1").alias("sketch1"), + ... sf.theta_sketch_agg("v2").alias("sketch2") + ... ) + >>> df.select(sf.theta_sketch_estimate(sf.theta_intersection(df.sketch1, "sketch2"))).show() + +-----------------------------------------------------------+ + |theta_sketch_estimate(theta_intersection(sketch1, sketch2))| + +-----------------------------------------------------------+ + | 3| + +-----------------------------------------------------------+ """ - fn = "kll_sketch_to_string_bigint" - return _invoke_function_over_columns(fn, col) + + fn = "theta_intersection" + return _invoke_function_over_columns(fn, col1, col2) @_try_remote_functions -def kll_sketch_to_string_float(col: "ColumnOrName") -> Column: +def theta_difference(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns a string with human readable summary information about the KLL float sketch. + Returns the set difference of two binary representations of Datasketches ThetaSketch + objects (elements in first sketch but not in second), using a Datasketches ANotB object. .. versionadded:: 4.1.0 + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + col2 : :class:`~pyspark.sql.Column` or column name + + Returns + ------- + :class:`~pyspark.sql.Column` + The binary representation of the difference ThetaSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.theta_union` + :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.theta_sketch_estimate` + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(1,4),(2,4),(3,5),(4,5)], "struct") + >>> df = df.agg( + ... sf.theta_sketch_agg("v1").alias("sketch1"), + ... sf.theta_sketch_agg("v2").alias("sketch2") + ... ) + >>> df.select(sf.theta_sketch_estimate(sf.theta_difference(df.sketch1, "sketch2"))).show() + +---------------------------------------------------------+ + |theta_sketch_estimate(theta_difference(sketch1, sketch2))| + +---------------------------------------------------------+ + | 3| + +---------------------------------------------------------+ + """ + + fn = "theta_difference" + return _invoke_function_over_columns(fn, col1, col2) + + +@_try_remote_functions +def tuple_sketch_estimate_double(col: "ColumnOrName") -> Column: + """ + Returns the estimated number of distinct keys from a Datasketches TupleSketch + with double summaries. + + .. versionadded:: 4.2.0 + Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. + The column containing a binary TupleSketch representation Returns ------- :class:`~pyspark.sql.Column` - A string representation of the sketch. - Returns a column that evaluates to a string. + The estimated cardinality. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_sketch_summary_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_to_string_float("sketch")).first()[0] - >>> "kll" in result.lower() - True + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +--------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| + +--------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------+ """ - fn = "kll_sketch_to_string_float" + fn = "tuple_sketch_estimate_double" return _invoke_function_over_columns(fn, col) @_try_remote_functions -def kll_sketch_to_string_double(col: "ColumnOrName") -> Column: +def tuple_sketch_estimate_integer(col: "ColumnOrName") -> Column: """ - Returns a string with human readable summary information about the KLL double sketch. + Returns the estimated number of distinct keys from a Datasketches TupleSketch + with integer summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. + The column containing a binary TupleSketch representation Returns ------- :class:`~pyspark.sql.Column` - A string representation of the sketch. - Returns a column that evaluates to a string. + The estimated cardinality. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_to_string_double("sketch")).first()[0] - >>> "kll" in result.lower() - True + >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_estimate_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +----------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| + +----------------------------------------------------------------------------+ + | 2.0| + +----------------------------------------------------------------------------+ """ - fn = "kll_sketch_to_string_double" + fn = "tuple_sketch_estimate_integer" return _invoke_function_over_columns(fn, col) @_try_remote_functions -def kll_sketch_get_n_bigint(col: "ColumnOrName") -> Column: +def tuple_sketch_summary_double( + col: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns the number of items collected in the KLL bigint sketch. + Returns the aggregated summary value from a Datasketches TupleSketch with double summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. - A column that evaluates to a binary. + The column containing a binary TupleSketch representation + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The count of items in the sketch. - Returns a column that evaluates to a long. + The aggregated summary value. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_n_bigint("sketch")).show() - +-------------------------------+ - |kll_sketch_get_n_bigint(sketch)| - +-------------------------------+ - | 5| - +-------------------------------+ + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_summary_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +------------------------------------------------------------------------------+ + |tuple_sketch_summary_double(tuple_sketch_agg_double(key, value, 12, sum), sum)| + +------------------------------------------------------------------------------+ + | 60.0| + +------------------------------------------------------------------------------+ """ - fn = "kll_sketch_get_n_bigint" - return _invoke_function_over_columns(fn, col) + fn = "tuple_sketch_summary_double" + if mode is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(mode)) @_try_remote_functions -def kll_sketch_get_n_float(col: "ColumnOrName") -> Column: +def tuple_sketch_summary_integer( + col: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns the number of items collected in the KLL float sketch. + Returns the aggregated summary value from a Datasketches TupleSketch with integer summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. + The column containing a binary TupleSketch representation + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The count of items in the sketch. - Returns a column that evaluates to a long. + The aggregated summary value. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_n_float("sketch")).show() - +------------------------------+ - |kll_sketch_get_n_float(sketch)| - +------------------------------+ - | 5| - +------------------------------+ + >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_summary_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +--------------------------------------------------------------------------------+ + |tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, value, 12, sum), sum)| + +--------------------------------------------------------------------------------+ + | 60| + +--------------------------------------------------------------------------------+ """ - fn = "kll_sketch_get_n_float" - return _invoke_function_over_columns(fn, col) + fn = "tuple_sketch_summary_integer" + if mode is None: + return _invoke_function_over_columns(fn, col) + else: + return _invoke_function_over_columns(fn, col, lit(mode)) @_try_remote_functions -def kll_sketch_get_n_double(col: "ColumnOrName") -> Column: +def tuple_sketch_theta_double(col: "ColumnOrName") -> Column: """ - Returns the number of items collected in the KLL double sketch. + Returns the theta value from a Datasketches TupleSketch with double summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. + The column containing a binary TupleSketch representation Returns ------- :class:`~pyspark.sql.Column` - The count of items in the sketch. - Returns a column that evaluates to a long. + The theta value (between 0.0 and 1.0). + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_n_double("sketch")).show() - +-------------------------------+ - |kll_sketch_get_n_double(sketch)| - +-------------------------------+ - | 5| - +-------------------------------+ + >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_theta_double( + ... sf.tuple_sketch_agg_double("key", "value"))).show() + +-----------------------------------------------------------------------+ + |tuple_sketch_theta_double(tuple_sketch_agg_double(key, value, 12, sum))| + +-----------------------------------------------------------------------+ + | 1.0| + +-----------------------------------------------------------------------+ """ - fn = "kll_sketch_get_n_double" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_columns("tuple_sketch_theta_double", col) @_try_remote_functions -def kll_sketch_merge_bigint(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def tuple_sketch_theta_integer(col: "ColumnOrName") -> Column: """ - Merges two KLL bigint sketch buffers together into one. + Returns the theta value from a Datasketches TupleSketch with integer summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The first KLL bigint sketch. - A column that evaluates to a binary. - right : :class:`~pyspark.sql.Column` or column name - The second KLL bigint sketch. - A column that evaluates to a binary. + col : :class:`~pyspark.sql.Column` or column name + The column containing a binary TupleSketch representation Returns ------- :class:`~pyspark.sql.Column` - The merged KLL sketch. - Returns a column that evaluates to a binary. + The theta value (between 0.0 and 1.0). + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_merge_bigint("sketch", "sketch")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> df = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) + >>> df.agg(sf.tuple_sketch_theta_integer( + ... sf.tuple_sketch_agg_integer("key", "value"))).show() + +-------------------------------------------------------------------------+ + |tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, value, 12, sum))| + +-------------------------------------------------------------------------+ + | 1.0| + +-------------------------------------------------------------------------+ """ - fn = "kll_sketch_merge_bigint" - return _invoke_function_over_columns(fn, left, right) + return _invoke_function_over_columns("tuple_sketch_theta_integer", col) @_try_remote_functions -def kll_sketch_merge_float(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def tuple_union_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Merges two KLL float sketch buffers together into one. + Returns the union of two Datasketches TupleSketch objects with double summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The first KLL float sketch. - A column that evaluates to a binary. - right : :class:`~pyspark.sql.Column` or column name - The second KLL float sketch. - A column that evaluates to a binary. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The merged KLL sketch. - Returns a column that evaluates to a binary. + The binary representation of the merged TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_agg_double` + :meth:`pyspark.sql.functions.tuple_intersection_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_merge_float("sketch", "sketch")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> df = spark.createDataFrame([(1, 10.0, 3, 30.0), (2, 20.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_double(df.sketch1, "sketch2"))).show() # noqa + +---------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_union_double(sketch1, sketch2, 12, sum))| + +---------------------------------------------------------------------------+ + | 4.0| + +---------------------------------------------------------------------------+ """ - fn = "kll_sketch_merge_float" - return _invoke_function_over_columns(fn, left, right) + fn = "tuple_union_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) @_try_remote_functions -def kll_sketch_merge_double(left: "ColumnOrName", right: "ColumnOrName") -> Column: +def tuple_union_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Merges two KLL double sketch buffers together into one. + Returns the union of two Datasketches TupleSketch objects with integer summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - The first KLL double sketch. - A column that evaluates to a binary. - right : :class:`~pyspark.sql.Column` or column name - The second KLL double sketch. - A column that evaluates to a binary. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The merged KLL sketch. - Returns a column that evaluates to a binary. + The binary representation of the merged TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_agg_integer` + :meth:`pyspark.sql.functions.tuple_intersection_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> result = sketch_df.select(sf.kll_sketch_merge_double("sketch", "sketch")).first()[0] - >>> result is not None and len(result) > 0 - True + >>> df = spark.createDataFrame([(1, 10, 3, 30), (2, 20, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_integer(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_union_integer(sketch1, sketch2, 12, sum))| + +-----------------------------------------------------------------------------+ + | 4.0| + +-----------------------------------------------------------------------------+ """ - fn = "kll_sketch_merge_double" - return _invoke_function_over_columns(fn, left, right) + fn = "tuple_union_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) @_try_remote_functions -def kll_sketch_get_quantile_bigint(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: +def tuple_intersection_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Extracts a quantile value from a KLL bigint sketch given an input rank value. - The rank can be a single value or an array. + Returns the intersection of two Datasketches TupleSketch objects with double summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. - A column that evaluates to a binary. - rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0). - A column that evaluates to a double or array. Must be a constant. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The quantile value(s). - Returns a column that evaluates to a long, or an array of longs if the rank - argument is an array. + The binary representation of the intersected TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_double` + :meth:`pyspark.sql.functions.tuple_intersection_agg_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_quantile_bigint("sketch", sf.lit(0.5))).show() - +-------------------------------------------+ - |kll_sketch_get_quantile_bigint(sketch, 0.5)| - +-------------------------------------------+ - | 3| - +-------------------------------------------+ + >>> df = spark.createDataFrame([(1, 10.0, 2, 20.0), (2, 20.0, 3, 30.0), (3, 30.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_double(df.sketch1, "sketch2"))).show() # noqa + +------------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_intersection_double(sketch1, sketch2, sum))| + +------------------------------------------------------------------------------+ + | 2.0| + +------------------------------------------------------------------------------+ """ - fn = "kll_sketch_get_quantile_bigint" - return _invoke_function_over_columns(fn, sketch, rank) + fn = "tuple_intersection_double" + if mode is None: + return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) @_try_remote_functions -def kll_sketch_get_quantile_float(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: +def tuple_intersection_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Extracts a quantile value from a KLL float sketch given an input rank value. - The rank can be a single value or an array. + Returns the intersection of two Datasketches TupleSketch objects with integer summaries. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. - rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0). - A column that evaluates to a double or array. Must be a constant. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The quantile value(s). - Returns a column that evaluates to a float, or an array of floats if the rank - argument is an array. + The binary representation of the intersected TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_integer` + :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_quantile_float("sketch", sf.lit(0.5))).show() - +------------------------------------------+ - |kll_sketch_get_quantile_float(sketch, 0.5)| - +------------------------------------------+ - | 3.0| - +------------------------------------------+ + >>> df = spark.createDataFrame([(1, 10, 2, 20), (2, 20, 3, 30), (3, 30, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_integer(df.sketch1, "sketch2"))).show() # noqa + +--------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_intersection_integer(sketch1, sketch2, sum))| + +--------------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------------+ """ - fn = "kll_sketch_get_quantile_float" - return _invoke_function_over_columns(fn, sketch, rank) + fn = "tuple_intersection_integer" + if mode is None: + return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) @_try_remote_functions -def kll_sketch_get_quantile_double(sketch: "ColumnOrName", rank: "ColumnOrName") -> Column: +def tuple_difference_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Extracts a quantile value from a KLL double sketch given an input rank value. - The rank can be a single value or an array. + Returns the set difference of two Datasketches TupleSketch objects with double summaries + (elements in first sketch but not in second). - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. - rank : :class:`~pyspark.sql.Column` or column name - The rank value(s) to extract (between 0.0 and 1.0). - A column that evaluates to a double or array. Must be a constant. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column Returns ------- :class:`~pyspark.sql.Column` - The quantile value(s). - Returns a column that evaluates to a double, or an array of doubles if the rank - argument is an array. + The binary representation of the difference TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.tuple_union_double` + :meth:`pyspark.sql.functions.tuple_intersection_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_quantile_double("sketch", sf.lit(0.5))).show() - +-------------------------------------------+ - |kll_sketch_get_quantile_double(sketch, 0.5)| - +-------------------------------------------+ - | 3.0| - +-------------------------------------------+ + >>> df = spark.createDataFrame([(1, 10.0, 4, 40.0), (2, 20.0, 4, 40.0), (3, 30.0, 5, 50.0), (4, 40.0, 5, 50.0)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_double(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_difference_double(sketch1, sketch2))| + +-----------------------------------------------------------------------+ + | 3.0| + +-----------------------------------------------------------------------+ """ - fn = "kll_sketch_get_quantile_double" - return _invoke_function_over_columns(fn, sketch, rank) + return _invoke_function_over_columns("tuple_difference_double", col1, col2) @_try_remote_functions -def kll_sketch_get_rank_bigint(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: +def tuple_difference_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Extracts a rank value from a KLL bigint sketch given an input quantile value. - The quantile can be a single value or an array. + Returns the set difference of two Datasketches TupleSketch objects with integer summaries + (elements in first sketch but not in second). - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL bigint sketch binary representation. - A column that evaluates to a binary. - quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup. - A column that evaluates to a long or array. Must be a constant. + col1 : :class:`~pyspark.sql.Column` or column name + The first TupleSketch column + col2 : :class:`~pyspark.sql.Column` or column name + The second TupleSketch column Returns ------- :class:`~pyspark.sql.Column` - The rank value(s) (between 0.0 and 1.0). - Returns a column that evaluates to a double, or an array of doubles if the quantile - argument is an array. + The binary representation of the difference TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.tuple_union_integer` + :meth:`pyspark.sql.functions.tuple_intersection_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,3,4,5], "INT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_bigint("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_rank_bigint("sketch", sf.lit(3))).show() - +-------------------------------------+ - |kll_sketch_get_rank_bigint(sketch, 3)| - +-------------------------------------+ - | 0.6| - +-------------------------------------+ + >>> df = spark.createDataFrame([(1, 10, 4, 40), (2, 20, 4, 40), (3, 30, 5, 50), (4, 40, 5, 50)], ["key1", "v1", "key2", "v2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_integer(df.sketch1, "sketch2"))).show() # noqa + +-------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_difference_integer(sketch1, sketch2))| + +-------------------------------------------------------------------------+ + | 3.0| + +-------------------------------------------------------------------------+ """ - fn = "kll_sketch_get_rank_bigint" - return _invoke_function_over_columns(fn, sketch, quantile) + return _invoke_function_over_columns("tuple_difference_integer", col1, col2) @_try_remote_functions -def kll_sketch_get_rank_float(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: +def tuple_difference_theta_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Extracts a rank value from a KLL float sketch given an input quantile value. - The quantile can be a single value or an array. + Subtracts a Datasketches ThetaSketch from a TupleSketch with double summaries + (elements in TupleSketch but not in ThetaSketch). - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL float sketch binary representation. - A column that evaluates to a binary. - quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup. - A column that evaluates to a float or array. Must be a constant. + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with double summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column Returns ------- :class:`~pyspark.sql.Column` - The rank value(s) (between 0.0 and 1.0). - Returns a column that evaluates to a double, or an array of doubles if the quantile - argument is an array. + The binary representation of the difference TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_theta_double` + :meth:`pyspark.sql.functions.tuple_intersection_theta_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "FLOAT") - >>> sketch_df = df.agg(sf.kll_sketch_agg_float("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_rank_float("sketch", sf.lit(3.0))).show() - +--------------------------------------+ - |kll_sketch_get_rank_float(sketch, 3.0)| - +--------------------------------------+ - | 0.6| - +--------------------------------------+ + >>> df = spark.createDataFrame([(5, 5.0, 4), (1, 1.0, 4), (2, 2.0, 5), (3, 3.0, 1)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_theta_double(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_difference_theta_double(sketch1, sketch2))| + +-----------------------------------------------------------------------------+ + | 2.0| + +-----------------------------------------------------------------------------+ """ - fn = "kll_sketch_get_rank_float" - return _invoke_function_over_columns(fn, sketch, quantile) + return _invoke_function_over_columns("tuple_difference_theta_double", col1, col2) @_try_remote_functions -def kll_sketch_get_rank_double(sketch: "ColumnOrName", quantile: "ColumnOrName") -> Column: +def tuple_difference_theta_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Extracts a rank value from a KLL double sketch given an input quantile value. - The quantile can be a single value or an array. + Subtracts a Datasketches ThetaSketch from a TupleSketch with integer summaries + (elements in TupleSketch but not in ThetaSketch). - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - sketch : :class:`~pyspark.sql.Column` or column name - The KLL double sketch binary representation. - A column that evaluates to a binary. - quantile : :class:`~pyspark.sql.Column` or column name - The quantile value(s) to lookup. - A column that evaluates to a double or array. Must be a constant. + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with integer summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column Returns ------- :class:`~pyspark.sql.Column` - The rank value(s) (between 0.0 and 1.0). - Returns a column that evaluates to a double, or an array of doubles if the quantile - argument is an array. + The binary representation of the difference TupleSketch. + + See Also + -------- + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` + :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_theta_integer` + :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1.0,2.0,3.0,4.0,5.0], "DOUBLE") - >>> sketch_df = df.agg(sf.kll_sketch_agg_double("value").alias("sketch")) - >>> sketch_df.select(sf.kll_sketch_get_rank_double("sketch", sf.lit(3.0))).show() - +---------------------------------------+ - |kll_sketch_get_rank_double(sketch, 3.0)| - +---------------------------------------+ - | 0.6| - +---------------------------------------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(5, 5, 4), (1, 1, 4), (2, 2, 5), (3, 3, 1)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_theta_integer(df.sketch1, "sketch2"))).show() # noqa + +-------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_difference_theta_integer(sketch1, sketch2))| + +-------------------------------------------------------------------------------+ + | 2.0| + +-------------------------------------------------------------------------------+ """ - fn = "kll_sketch_get_rank_double" - return _invoke_function_over_columns(fn, sketch, quantile) + return _invoke_function_over_columns("tuple_difference_theta_integer", col1, col2) @_try_remote_functions -def theta_sketch_estimate(col: "ColumnOrName") -> Column: +def tuple_intersection_theta_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns the estimated number of unique values given the binary representation - of a Datasketches ThetaSketch. + Intersects a Datasketches TupleSketch with double summaries with a ThetaSketch. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name + col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with double summaries + col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The estimated number of unique values for the ThetaSketch. + The binary representation of the intersected TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_intersection` - :meth:`pyspark.sql.functions.theta_difference` - :meth:`pyspark.sql.functions.theta_union_agg` - :meth:`pyspark.sql.functions.theta_intersection_agg` + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` :meth:`pyspark.sql.functions.theta_sketch_agg` + :meth:`pyspark.sql.functions.tuple_union_theta_double` + :meth:`pyspark.sql.functions.tuple_intersection_agg_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([1,2,2,3], "INT") - >>> df.agg(sf.theta_sketch_estimate(sf.theta_sketch_agg("value"))).show() - +--------------------------------------------------+ - |theta_sketch_estimate(theta_sketch_agg(value, 12))| - +--------------------------------------------------+ - | 3| - +--------------------------------------------------+ + >>> df = spark.createDataFrame([(1, 1.0, 1), (2, 2.0, 2), (3, 3.0, 4)], ["key1", "v1", "key2"]) # noqa + >>> df = df.agg( + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") + ... ) + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_theta_double(df.sketch1, "sketch2"))).show() # noqa + +------------------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_intersection_theta_double(sketch1, sketch2, sum))| + +------------------------------------------------------------------------------------+ + | 2.0| + +------------------------------------------------------------------------------------+ """ - - fn = "theta_sketch_estimate" - return _invoke_function_over_columns(fn, col) + fn = "tuple_intersection_theta_double" + if mode is None: + return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) @_try_remote_functions -def theta_union( - col1: "ColumnOrName", col2: "ColumnOrName", lgNomEntries: Optional[Union[int, Column]] = None +def tuple_intersection_theta_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + mode: Optional[Union[str, Column]] = None, ) -> Column: """ - Merges two binary representations of Datasketches ThetaSketch objects, using a - Datasketches Union object. + Intersects a Datasketches TupleSketch with integer summaries with a ThetaSketch. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with integer summaries col2 : :class:`~pyspark.sql.Column` or column name - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries for the union operation - (must be between 4 and 26, defaults to 12) + The ThetaSketch column + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged ThetaSketch. + The binary representation of the intersected TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.theta_union_agg` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + :meth:`pyspark.sql.functions.tuple_union_theta_integer` + :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,4),(2,5),(2,5),(3,6)], "struct") + >>> df = spark.createDataFrame([(1, 1, 1), (2, 2, 2), (3, 3, 4)], ["key1", "v1", "key2"]) # noqa >>> df = df.agg( - ... sf.theta_sketch_agg("v1").alias("sketch1"), - ... sf.theta_sketch_agg("v2").alias("sketch2") + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") ... ) - >>> df.select(sf.theta_sketch_estimate(sf.theta_union(df.sketch1, "sketch2"))).show() - +--------------------------------------------------------+ - |theta_sketch_estimate(theta_union(sketch1, sketch2, 12))| - +--------------------------------------------------------+ - | 6| - +--------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_theta_integer(df.sketch1, "sketch2"))).show() # noqa + +--------------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_intersection_theta_integer(sketch1, sketch2, sum))| + +--------------------------------------------------------------------------------------+ + | 2.0| + +--------------------------------------------------------------------------------------+ """ - - fn = "theta_union" - if lgNomEntries is not None: - return _invoke_function_over_columns( - fn, - col1, - col2, - lit(lgNomEntries), - ) - else: + fn = "tuple_intersection_theta_integer" + if mode is None: return _invoke_function_over_columns(fn, col1, col2) + else: + return _invoke_function_over_columns(fn, col1, col2, lit(mode)) @_try_remote_functions -def theta_intersection(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def tuple_union_theta_double( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns the intersection of two binary representations of Datasketches ThetaSketch - objects, using a Datasketches Intersection object. + Merges a Datasketches TupleSketch with double summaries with a ThetaSketch. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with double summaries col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected ThetaSketch. + The binary representation of the merged TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.theta_intersection_agg` + :meth:`pyspark.sql.functions.tuple_sketch_agg_double` :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + :meth:`pyspark.sql.functions.tuple_union_agg_double` + :meth:`pyspark.sql.functions.tuple_intersection_theta_double` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,1),(2,2),(3,2),(3,3)], "struct") + >>> df = spark.createDataFrame([(1, 10.0, 3), (2, 20.0, 4)], ["key1", "v1", "key2"]) # noqa >>> df = df.agg( - ... sf.theta_sketch_agg("v1").alias("sketch1"), - ... sf.theta_sketch_agg("v2").alias("sketch2") + ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") ... ) - >>> df.select(sf.theta_sketch_estimate(sf.theta_intersection(df.sketch1, "sketch2"))).show() - +-----------------------------------------------------------+ - |theta_sketch_estimate(theta_intersection(sketch1, sketch2))| - +-----------------------------------------------------------+ - | 3| - +-----------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_theta_double(df.sketch1, "sketch2"))).show() # noqa + +---------------------------------------------------------------------------------+ + |tuple_sketch_estimate_double(tuple_union_theta_double(sketch1, sketch2, 12, sum))| + +---------------------------------------------------------------------------------+ + | 4.0| + +---------------------------------------------------------------------------------+ """ + fn = "tuple_union_theta_double" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) - fn = "theta_intersection" - return _invoke_function_over_columns(fn, col1, col2) + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) @_try_remote_functions -def theta_difference(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def tuple_union_theta_integer( + col1: "ColumnOrName", + col2: "ColumnOrName", + lgNomEntries: Optional[Union[int, Column]] = None, + mode: Optional[Union[str, Column]] = None, +) -> Column: """ - Returns the set difference of two binary representations of Datasketches ThetaSketch - objects (elements in first sketch but not in second), using a Datasketches ANotB object. + Merges a Datasketches TupleSketch with integer summaries with a ThetaSketch. - .. versionadded:: 4.1.0 + .. versionadded:: 4.2.0 Parameters ---------- col1 : :class:`~pyspark.sql.Column` or column name + The TupleSketch column with integer summaries col2 : :class:`~pyspark.sql.Column` or column name + The ThetaSketch column + lgNomEntries : :class:`~pyspark.sql.Column` or int, optional + The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) + A column that evaluates to an integer. + mode : :class:`~pyspark.sql.Column` or str, optional + The summary mode: "sum" (default), "min", "max", or "alwaysone" Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the difference ThetaSketch. + The binary representation of the merged TupleSketch. See Also -------- - :meth:`pyspark.sql.functions.theta_union` - :meth:`pyspark.sql.functions.theta_intersection` + :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.theta_sketch_estimate` + :meth:`pyspark.sql.functions.tuple_union_agg_integer` + :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1,4),(2,4),(3,5),(4,5)], "struct") + >>> df = spark.createDataFrame([(1, 10, 3), (2, 20, 4)], ["key1", "v1", "key2"]) # noqa >>> df = df.agg( - ... sf.theta_sketch_agg("v1").alias("sketch1"), - ... sf.theta_sketch_agg("v2").alias("sketch2") + ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), + ... sf.theta_sketch_agg("key2").alias("sketch2") ... ) - >>> df.select(sf.theta_sketch_estimate(sf.theta_difference(df.sketch1, "sketch2"))).show() - +---------------------------------------------------------+ - |theta_sketch_estimate(theta_difference(sketch1, sketch2))| - +---------------------------------------------------------+ - | 3| - +---------------------------------------------------------+ + >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_theta_integer(df.sketch1, "sketch2"))).show() # noqa + +-----------------------------------------------------------------------------------+ + |tuple_sketch_estimate_integer(tuple_union_theta_integer(sketch1, sketch2, 12, sum))| + +-----------------------------------------------------------------------------------+ + | 4.0| + +-----------------------------------------------------------------------------------+ """ + fn = "tuple_union_theta_integer" + _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) + _mode = lit("sum") if mode is None else lit(mode) + + return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) - fn = "theta_difference" - return _invoke_function_over_columns(fn, col1, col2) + +# ---------------------- Predicates functions ------------------------------ @_try_remote_functions -def tuple_sketch_estimate_double(col: "ColumnOrName") -> Column: +def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns the estimated number of distinct keys from a Datasketches TupleSketch - with double summaries. + Returns `col2` if `col1` is null, or `col1` otherwise. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - - Returns - ------- - :class:`~pyspark.sql.Column` - The estimated cardinality. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_sketch_summary_double` + col1 : :class:`~pyspark.sql.Column` or str + col2 : :class:`~pyspark.sql.Column` or str Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +--------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_sketch_agg_double(key, value, 12, sum))| - +--------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) + >>> df.select(sf.ifnull(df.e, sf.lit(8))).show() + +------------+ + |ifnull(e, 8)| + +------------+ + | 8| + | 1| + +------------+ """ - fn = "tuple_sketch_estimate_double" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_columns("ifnull", col1, col2) @_try_remote_functions -def tuple_sketch_estimate_integer(col: "ColumnOrName") -> Column: +def isnotnull(col: "ColumnOrName") -> Column: """ - Returns the estimated number of distinct keys from a Datasketches TupleSketch - with integer summaries. + Returns true if `col` is not null, or false otherwise. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - - Returns - ------- - :class:`~pyspark.sql.Column` - The estimated cardinality. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_sketch_summary_integer` + :meth:`pyspark.sql.functions.isnan` + :meth:`pyspark.sql.functions.isnull` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_estimate_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +----------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_sketch_agg_integer(key, value, 12, sum))| - +----------------------------------------------------------------------------+ - | 2.0| - +----------------------------------------------------------------------------+ + >>> df = spark.createDataFrame([(None,), (1,)], ["e"]) + >>> df.select('*', sf.isnotnull(df.e)).show() + +----+---------------+ + | e|(e IS NOT NULL)| + +----+---------------+ + |NULL| false| + | 1| true| + +----+---------------+ + + >>> df.select('*', sf.isnotnull('e')).show() + +----+---------------+ + | e|(e IS NOT NULL)| + +----+---------------+ + |NULL| false| + | 1| true| + +----+---------------+ """ - fn = "tuple_sketch_estimate_integer" - return _invoke_function_over_columns(fn, col) + return _invoke_function_over_columns("isnotnull", col) @_try_remote_functions -def tuple_sketch_summary_double( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def equal_null(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ - Returns the aggregated summary value from a Datasketches TupleSketch with double summaries. + Returns same result as the EQUAL(=) operator for non-null operands, + but returns true if both are null, false if one of them is null. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" - - Returns - ------- - :class:`~pyspark.sql.Column` - The aggregated summary value. + col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. + col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. - See Also + Examples -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) + >>> df.select('*', sf.equal_null(df.a, df.b)).show() + +----+----+----------------+ + | a| b|equal_null(a, b)| + +----+----+----------------+ + |NULL|NULL| true| + | 1| 9| false| + +----+----+----------------+ + + >>> df.select('*', sf.equal_null('a', 'b')).show() + +----+----+----------------+ + | a| b|equal_null(a, b)| + +----+----+----------------+ + |NULL|NULL| true| + | 1| 9| false| + +----+----+----------------+ + """ + return _invoke_function_over_columns("equal_null", col1, col2) + + +@_try_remote_functions +def nullif(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """ + Returns null if `col1` equals to `col2`, or `col1` otherwise. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. + col2 : :class:`~pyspark.sql.Column` or column name + A column of any orderable type. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0), (2, 30.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_summary_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +------------------------------------------------------------------------------+ - |tuple_sketch_summary_double(tuple_sketch_agg_double(key, value, 12, sum), sum)| - +------------------------------------------------------------------------------+ - | 60.0| - +------------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(None, None,), (1, 9,)], ["a", "b"]) + >>> df.select('*', sf.nullif(df.a, df.b)).show() + +----+----+------------+ + | a| b|nullif(a, b)| + +----+----+------------+ + |NULL|NULL| NULL| + | 1| 9| 1| + +----+----+------------+ + + >>> df.select('*', sf.nullif('a', 'b')).show() + +----+----+------------+ + | a| b|nullif(a, b)| + +----+----+------------+ + |NULL|NULL| NULL| + | 1| 9| 1| + +----+----+------------+ """ - fn = "tuple_sketch_summary_double" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + return _invoke_function_over_columns("nullif", col1, col2) @_try_remote_functions -def tuple_sketch_summary_integer( - col: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def nullifzero(col: "ColumnOrName") -> Column: """ - Returns the aggregated summary value from a Datasketches TupleSketch with integer summaries. + Returns null if `col` is equal to zero, or `col` otherwise. - .. versionadded:: 4.2.0 + .. versionadded:: 4.0.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + A column that evaluates to a numeric. - Returns - ------- - :class:`~pyspark.sql.Column` - The aggregated summary value. + Examples + -------- + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(0,), (1,)], ["a"]) + >>> df.select('*', sf.nullifzero(df.a)).show() + +---+-------------+ + | a|nullifzero(a)| + +---+-------------+ + | 0| NULL| + | 1| 1| + +---+-------------+ + + >>> df.select('*', sf.nullifzero('a')).show() + +---+-------------+ + | a|nullifzero(a)| + +---+-------------+ + | 0| NULL| + | 1| 1| + +---+-------------+ + """ + return _invoke_function_over_columns("nullifzero", col) + + +@_try_remote_functions +def nvl(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: + """ + Returns `col2` if `col1` is null, or `col1` otherwise. + + .. versionadded:: 3.5.0 + + Parameters + ---------- + col1 : :class:`~pyspark.sql.Column` or column name + col2 : :class:`~pyspark.sql.Column` or column name See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` + :meth:`pyspark.sql.functions.nvl2` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20), (2, 30)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_summary_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +--------------------------------------------------------------------------------+ - |tuple_sketch_summary_integer(tuple_sketch_agg_integer(key, value, 12, sum), sum)| - +--------------------------------------------------------------------------------+ - | 60| - +--------------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(None, 8,), (1, 9,)], ["a", "b"]) + >>> df.select('*', sf.nvl(df.a, df.b)).show() + +----+---+---------+ + | a| b|nvl(a, b)| + +----+---+---------+ + |NULL| 8| 8| + | 1| 9| 1| + +----+---+---------+ + + >>> df.select('*', sf.nvl('a', 'b')).show() + +----+---+---------+ + | a| b|nvl(a, b)| + +----+---+---------+ + |NULL| 8| 8| + | 1| 9| 1| + +----+---+---------+ """ - fn = "tuple_sketch_summary_integer" - if mode is None: - return _invoke_function_over_columns(fn, col) - else: - return _invoke_function_over_columns(fn, col, lit(mode)) + return _invoke_function_over_columns("nvl", col1, col2) @_try_remote_functions -def tuple_sketch_theta_double(col: "ColumnOrName") -> Column: +def nvl2(col1: "ColumnOrName", col2: "ColumnOrName", col3: "ColumnOrName") -> Column: """ - Returns the theta value from a Datasketches TupleSketch with double summaries. + Returns `col2` if `col1` is not null, or `col3` otherwise. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - - Returns - ------- - :class:`~pyspark.sql.Column` - The theta value (between 0.0 and 1.0). + col1 : :class:`~pyspark.sql.Column` or column name + col2 : :class:`~pyspark.sql.Column` or column name + col3 : :class:`~pyspark.sql.Column` or column name See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_double` + :meth:`pyspark.sql.functions.nvl` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0), (2, 20.0)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_theta_double( - ... sf.tuple_sketch_agg_double("key", "value"))).show() - +-----------------------------------------------------------------------+ - |tuple_sketch_theta_double(tuple_sketch_agg_double(key, value, 12, sum))| - +-----------------------------------------------------------------------+ - | 1.0| - +-----------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(None, 8, 6,), (1, 9, 9,)], ["a", "b", "c"]) + >>> df.select('*', sf.nvl2(df.a, df.b, df.c)).show() + +----+---+---+-------------+ + | a| b| c|nvl2(a, b, c)| + +----+---+---+-------------+ + |NULL| 8| 6| 6| + | 1| 9| 9| 9| + +----+---+---+-------------+ + + >>> df.select('*', sf.nvl2('a', 'b', 'c')).show() + +----+---+---+-------------+ + | a| b| c|nvl2(a, b, c)| + +----+---+---+-------------+ + |NULL| 8| 6| 6| + | 1| 9| 9| 9| + +----+---+---+-------------+ """ - return _invoke_function_over_columns("tuple_sketch_theta_double", col) + return _invoke_function_over_columns("nvl2", col1, col2, col3) @_try_remote_functions -def tuple_sketch_theta_integer(col: "ColumnOrName") -> Column: +def zeroifnull(col: "ColumnOrName") -> Column: """ - Returns the theta value from a Datasketches TupleSketch with integer summaries. + Returns zero if `col` is null, or `col` otherwise. - .. versionadded:: 4.2.0 + .. versionadded:: 4.0.0 Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column containing a binary TupleSketch representation - - Returns - ------- - :class:`~pyspark.sql.Column` - The theta value (between 0.0 and 1.0). - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_sketch_estimate_integer` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10), (2, 20)], ["key", "value"]) - >>> df.agg(sf.tuple_sketch_theta_integer( - ... sf.tuple_sketch_agg_integer("key", "value"))).show() - +-------------------------------------------------------------------------+ - |tuple_sketch_theta_integer(tuple_sketch_agg_integer(key, value, 12, sum))| - +-------------------------------------------------------------------------+ - | 1.0| - +-------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([(None,), (1,)], ["a"]) + >>> df.select('*', sf.zeroifnull(df.a)).show() + +----+-------------+ + | a|zeroifnull(a)| + +----+-------------+ + |NULL| 0| + | 1| 1| + +----+-------------+ + + >>> df.select('*', sf.zeroifnull('a')).show() + +----+-------------+ + | a|zeroifnull(a)| + +----+-------------+ + |NULL| 0| + | 1| 1| + +----+-------------+ """ - return _invoke_function_over_columns("tuple_sketch_theta_integer", col) + return _invoke_function_over_columns("zeroifnull", col) @_try_remote_functions -def tuple_union_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, +def hmac( + key: "ColumnOrName", + message: "ColumnOrName", + algorithm: Optional["ColumnOrName"] = None, ) -> Column: """ - Returns the union of two Datasketches TupleSketch objects with double summaries. + Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the + given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with :func:`hex` or + :func:`base64` for a textual value. The default algorithm is 'SHA-256'. - .. versionadded:: 4.2.0 + .. versionadded:: 4.3.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + key : :class:`~pyspark.sql.Column` or column name + The secret key, as a binary value. + message : :class:`~pyspark.sql.Column` or column name + The message to authenticate, as a binary value. + algorithm : :class:`~pyspark.sql.Column` or column name, optional + The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. + The default is SHA-256. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. - - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_agg_double` - :meth:`pyspark.sql.functions.tuple_intersection_double` + A new column that contains the raw HMAC bytes. Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 3, 30.0), (2, 20.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_double(df.sketch1, "sketch2"))).show() # noqa - +---------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_union_double(sketch1, sketch2, 12, sum))| - +---------------------------------------------------------------------------+ - | 4.0| - +---------------------------------------------------------------------------+ - """ - fn = "tuple_union_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) + Example 1: Compute the HMAC with the default SHA-256 algorithm. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) + >>> df.select(sf.hex(sf.hmac(df.key, df.message))).show(truncate=False) + +----------------------------------------------------------------+ + |hex(hmac(key, message, SHA-256)) | + +----------------------------------------------------------------+ + |6E9EF29B75FFFC5B7ABAE527D58FDADB2FE42E7219011976917343065F58ED4A| + +----------------------------------------------------------------+ + + Example 2: Compute the HMAC with an explicit algorithm. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([("key", "message")], ["key", "message"]) + >>> df.select(sf.hex(sf.hmac(df.key, df.message, sf.lit("SHA-1")))).show(truncate=False) + +----------------------------------------+ + |hex(hmac(key, message, SHA-1)) | + +----------------------------------------+ + |2088DF74D5F2146B48146CAF4965377E9D0BE3A4| + +----------------------------------------+ + """ + if algorithm is None: + return _invoke_function_over_columns("hmac", key, message) + else: + return _invoke_function_over_columns("hmac", key, message, algorithm) @_try_remote_functions -def tuple_union_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, +def aes_encrypt( + input: "ColumnOrName", + key: "ColumnOrName", + mode: Optional["ColumnOrName"] = None, + padding: Optional["ColumnOrName"] = None, + iv: Optional["ColumnOrName"] = None, + aad: Optional["ColumnOrName"] = None, ) -> Column: """ - Returns the union of two Datasketches TupleSketch objects with integer summaries. + Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. + Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, + `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization + vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 + bytes for GCM. If not provided, a random vector will be generated and prepended to the + output. Optional additional authenticated data (AAD) is only supported for GCM. If provided + for encryption, the identical AAD value must be provided for decryption. The default mode is + GCM. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. + input : :class:`~pyspark.sql.Column` or column name + The binary value to encrypt. + A column that evaluates to a binary. + key : :class:`~pyspark.sql.Column` or column name + The passphrase to use to encrypt the data. + A column that evaluates to a binary. mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + GCM, CBC. + A column that evaluates to a string. + padding : :class:`~pyspark.sql.Column` or column name, optional + Specifies how to pad messages whose length is not a multiple of the block size. Valid + values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + for CBC. + A column that evaluates to a string. + iv : :class:`~pyspark.sql.Column` or column name, optional + Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + "". 16-byte array for CBC mode. 12-byte array for GCM mode. + A column that evaluates to a binary. + aad : :class:`~pyspark.sql.Column` or column name, optional + Optional additional authenticated data. Only supported for GCM mode. This can be any + free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. + A new column that contains an encrypted value. + Returns a column that evaluates to a binary. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_agg_integer` - :meth:`pyspark.sql.functions.tuple_intersection_integer` + :meth:`pyspark.sql.functions.aes_decrypt` + :meth:`pyspark.sql.functions.try_aes_decrypt` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 3, 30), (2, 20, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") + + Example 1: Encrypt data with key, mode, padding, iv and aad. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "000000000000000000000000", "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "iv", "aad"] ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_integer(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_union_integer(sketch1, sketch2, 12, sum))| - +-----------------------------------------------------------------------------+ - | 4.0| - +-----------------------------------------------------------------------------+ - """ - fn = "tuple_union_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) + >>> df.select(sf.base64(sf.aes_encrypt( + ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex")), df.aad) + ... )).show(truncate=False) + +-----------------------------------------------------------------------+ + |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), aad))| + +-----------------------------------------------------------------------+ + |AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4 | + +-----------------------------------------------------------------------+ - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) + Example 2: Encrypt data with key, mode, padding and iv. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark", "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "000000000000000000000000", "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "iv", "aad"] + ... ) + >>> df.select(sf.base64(sf.aes_encrypt( + ... df.input, df.key, "mode", df.padding, sf.to_binary(df.iv, sf.lit("hex"))) + ... )).show(truncate=False) + +--------------------------------------------------------------------+ + |base64(aes_encrypt(input, key, mode, padding, to_binary(iv, hex), ))| + +--------------------------------------------------------------------+ + |AAAAAAAAAAAAAAAAQiYi+sRNYDAOTjdSEcYBFsAWPL1f | + +--------------------------------------------------------------------+ -@_try_remote_functions -def tuple_intersection_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: - """ - Returns the intersection of two Datasketches TupleSketch objects with double summaries. + Example 3: Encrypt data with key, mode and padding. - .. versionadded:: 4.2.0 + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark SQL", "1234567890abcdef", "ECB", "PKCS",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode", df.padding), + ... df.key, df.mode, df.padding + ... ).cast("STRING")).show(truncate=False) + +---------------------------------------------------------------------------------------------+ + |CAST(aes_decrypt(aes_encrypt(input, key, mode, padding, , ), key, mode, padding, ) AS STRING)| + +---------------------------------------------------------------------------------------------+ + |Spark SQL | + +---------------------------------------------------------------------------------------------+ - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + Example 4: Encrypt data with key and mode. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark SQL", "0000111122223333", "ECB",)], + ... ["input", "key", "mode"] + ... ) + >>> df.select(sf.aes_decrypt(sf.aes_encrypt(df.input, df.key, "mode"), + ... df.key, df.mode + ... ).cast("STRING")).show(truncate=False) + +---------------------------------------------------------------------------------------------+ + |CAST(aes_decrypt(aes_encrypt(input, key, mode, DEFAULT, , ), key, mode, DEFAULT, ) AS STRING)| + +---------------------------------------------------------------------------------------------+ + |Spark SQL | + +---------------------------------------------------------------------------------------------+ - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_double` - :meth:`pyspark.sql.functions.tuple_intersection_agg_double` + Example 5: Encrypt data with key. - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 2, 20.0), (2, 20.0, 3, 30.0), (3, 30.0, 4, 40.0)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "Spark SQL", "abcdefghijklmnop",)], + ... ["input", "key"] ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_double(df.sketch1, "sketch2"))).show() # noqa - +------------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_intersection_double(sketch1, sketch2, sum))| - +------------------------------------------------------------------------------+ - | 2.0| - +------------------------------------------------------------------------------+ + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(sf.base64(sf.aes_encrypt(df.input, df.key))), df.key + ... ).cast("STRING")).show(truncate=False) + +-------------------------------------------------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(base64(aes_encrypt(input, key, GCM, DEFAULT, , ))), key, GCM, DEFAULT, ) AS STRING)| + +-------------------------------------------------------------------------------------------------------------+ + |Spark SQL | + +-------------------------------------------------------------------------------------------------------------+ """ - fn = "tuple_intersection_double" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) - else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + _mode = lit("GCM") if mode is None else mode + _padding = lit("DEFAULT") if padding is None else padding + _iv = lit("") if iv is None else iv + _aad = lit("") if aad is None else aad + return _invoke_function_over_columns("aes_encrypt", input, key, _mode, _padding, _iv, _aad) @_try_remote_functions -def tuple_intersection_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, +def aes_decrypt( + input: "ColumnOrName", + key: "ColumnOrName", + mode: Optional["ColumnOrName"] = None, + padding: Optional["ColumnOrName"] = None, + aad: Optional["ColumnOrName"] = None, ) -> Column: """ - Returns the intersection of two Datasketches TupleSketch objects with integer summaries. + Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, + 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', + 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is + only supported for GCM. If provided for encryption, the identical AAD value must be provided + for decryption. The default mode is GCM. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + input : :class:`~pyspark.sql.Column` or column name + The binary value to decrypt. + A column that evaluates to a binary. + key : :class:`~pyspark.sql.Column` or column name + The passphrase to use to decrypt the data. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or column name, optional + Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + GCM, CBC. + A column that evaluates to a string. + padding : :class:`~pyspark.sql.Column` or column name, optional + Specifies how to pad messages whose length is not a multiple of the block size. Valid + values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + for CBC. + A column that evaluates to a string. + aad : :class:`~pyspark.sql.Column` or column name, optional + Optional additional authenticated data. Only supported for GCM mode. This can be any + free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + A new column that contains a decrypted value. + Returns a column that evaluates to a binary. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_integer` - :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` + :meth:`pyspark.sql.functions.aes_encrypt` + :meth:`pyspark.sql.functions.try_aes_decrypt` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 2, 20), (2, 20, 3, 30), (3, 30, 4, 40)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_integer(df.sketch1, "sketch2"))).show() # noqa - +--------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_intersection_integer(sketch1, sketch2, sum))| - +--------------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------------+ - """ - fn = "tuple_intersection_integer" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) - else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + Example 1: Decrypt data with key, mode, padding and aad. -@_try_remote_functions -def tuple_difference_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """ - Returns the set difference of two Datasketches TupleSketch objects with double summaries - (elements in first sketch but not in second). + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", + ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "aad"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad + ... ).cast("STRING")).show(truncate=False) + +---------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| + +---------------------------------------------------------------------+ + |Spark | + +---------------------------------------------------------------------+ - .. versionadded:: 4.2.0 + Example 2: Decrypt data with key, mode and padding. - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding + ... ).cast("STRING")).show(truncate=False) + +------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| + +------------------------------------------------------------------+ + |Spark | + +------------------------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. + Example 3: Decrypt data with key and mode. - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.tuple_union_double` - :meth:`pyspark.sql.functions.tuple_intersection_double` + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode" + ... ).cast("STRING")).show(truncate=False) + +------------------------------------------------------------------+ + |CAST(aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| + +------------------------------------------------------------------+ + |Spark | + +------------------------------------------------------------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 4, 40.0), (2, 20.0, 4, 40.0), (3, 30.0, 5, 50.0), (4, 40.0, 5, 50.0)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_double("key2", "v2").alias("sketch2") + Example 4: Decrypt data with key. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", + ... "0000111122223333",)], + ... ["input", "key"] ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_double(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_difference_double(sketch1, sketch2))| - +-----------------------------------------------------------------------+ - | 3.0| - +-----------------------------------------------------------------------+ + >>> df.select(sf.aes_decrypt( + ... sf.unhex(df.input), df.key + ... ).cast("STRING")).show(truncate=False) + +--------------------------------------------------------------+ + |CAST(aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| + +--------------------------------------------------------------+ + |Spark | + +--------------------------------------------------------------+ """ - return _invoke_function_over_columns("tuple_difference_double", col1, col2) + _mode = lit("GCM") if mode is None else mode + _padding = lit("DEFAULT") if padding is None else padding + _aad = lit("") if aad is None else aad + return _invoke_function_over_columns("aes_decrypt", input, key, _mode, _padding, _aad) @_try_remote_functions -def tuple_difference_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def try_aes_decrypt( + input: "ColumnOrName", + key: "ColumnOrName", + mode: Optional["ColumnOrName"] = None, + padding: Optional["ColumnOrName"] = None, + aad: Optional["ColumnOrName"] = None, +) -> Column: """ - Returns the set difference of two Datasketches TupleSketch objects with integer summaries - (elements in first sketch but not in second). + This is a special version of `aes_decrypt` that performs the same operation, + but returns a NULL value instead of raising an error if the decryption cannot be performed. + Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, + 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', + 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is + only supported for GCM. If provided for encryption, the identical AAD value must be provided + for decryption. The default mode is GCM. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The first TupleSketch column - col2 : :class:`~pyspark.sql.Column` or column name - The second TupleSketch column + input : :class:`~pyspark.sql.Column` or column name + The binary value to decrypt. + A column that evaluates to a binary. + key : :class:`~pyspark.sql.Column` or column name + The passphrase to use to decrypt the data. + A column that evaluates to a binary. + mode : :class:`~pyspark.sql.Column` or column name, optional + Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + GCM, CBC. + A column that evaluates to a string. + padding : :class:`~pyspark.sql.Column` or column name, optional + Specifies how to pad messages whose length is not a multiple of the block size. Valid + values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + for CBC. + A column that evaluates to a string. + aad : :class:`~pyspark.sql.Column` or column name, optional + Optional additional authenticated data. Only supported for GCM mode. This can be any + free-form input and must be provided for both encryption and decryption. + A column that evaluates to a binary. Returns ------- :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. + A new column that contains a decrypted value or a NULL value. + Returns a column that evaluates to a binary. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.tuple_union_integer` - :meth:`pyspark.sql.functions.tuple_intersection_integer` + :meth:`pyspark.sql.functions.aes_encrypt` + :meth:`pyspark.sql.functions.aes_decrypt` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 4, 40), (2, 20, 4, 40), (3, 30, 5, 50), (4, 40, 5, 50)], ["key1", "v1", "key2", "v2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.tuple_sketch_agg_integer("key2", "v2").alias("sketch2") + + Example 1: Decrypt data with key, mode, padding and aad. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", + ... "abcdefghijklmnop12345678ABCDEFGH", "GCM", "DEFAULT", + ... "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "aad"] ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_integer(df.sketch1, "sketch2"))).show() # noqa + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad + ... ).cast("STRING")).show(truncate=False) +-------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_difference_integer(sketch1, sketch2))| + |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| +-------------------------------------------------------------------------+ - | 3.0| + |Spark | +-------------------------------------------------------------------------+ - """ - return _invoke_function_over_columns("tuple_difference_integer", col1, col2) + Example 2: Failed to decrypt data with key, mode, padding and aad. -@_try_remote_functions -def tuple_difference_theta_double(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: - """ - Subtracts a Datasketches ThetaSketch from a TupleSketch with double summaries - (elements in TupleSketch but not in ThetaSketch). + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAQiYi+sTLm7KD9UcZ2nlRdYDe/PX4", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT", + ... "This is an AAD mixed into the input",)], + ... ["input", "key", "mode", "padding", "aad"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding, df.aad + ... ).cast("STRING")).show(truncate=False) + +-------------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, aad) AS STRING)| + +-------------------------------------------------------------------------+ + |NULL | + +-------------------------------------------------------------------------+ - .. versionadded:: 4.2.0 + Example 3: Decrypt data with key, mode and padding. - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with double summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode", df.padding + ... ).cast("STRING")).show(truncate=False) + +----------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unbase64(input), key, mode, padding, ) AS STRING)| + +----------------------------------------------------------------------+ + |Spark | + +----------------------------------------------------------------------+ - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. + Example 4: Decrypt data with key and mode. - See Also - -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_double` - :meth:`pyspark.sql.functions.tuple_intersection_theta_double` + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "AAAAAAAAAAAAAAAAAAAAAPSd4mWyMZ5mhvjiAPQJnfg=", + ... "abcdefghijklmnop12345678ABCDEFGH", "CBC", "DEFAULT",)], + ... ["input", "key", "mode", "padding"] + ... ) + >>> df.select(sf.try_aes_decrypt( + ... sf.unbase64(df.input), df.key, "mode" + ... ).cast("STRING")).show(truncate=False) + +----------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unbase64(input), key, mode, DEFAULT, ) AS STRING)| + +----------------------------------------------------------------------+ + |Spark | + +----------------------------------------------------------------------+ - Examples - -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(5, 5.0, 4), (1, 1.0, 4), (2, 2.0, 5), (3, 3.0, 1)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") + Example 5: Decrypt data with key. + + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([( + ... "83F16B2AA704794132802D248E6BFD4E380078182D1544813898AC97E709B28A94", + ... "0000111122223333",)], + ... ["input", "key"] ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_difference_theta_double(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_difference_theta_double(sketch1, sketch2))| - +-----------------------------------------------------------------------------+ - | 2.0| - +-----------------------------------------------------------------------------+ + >>> df.select(sf.try_aes_decrypt( + ... sf.unhex(df.input), df.key + ... ).cast("STRING")).show(truncate=False) + +------------------------------------------------------------------+ + |CAST(try_aes_decrypt(unhex(input), key, GCM, DEFAULT, ) AS STRING)| + +------------------------------------------------------------------+ + |Spark | + +------------------------------------------------------------------+ """ - return _invoke_function_over_columns("tuple_difference_theta_double", col1, col2) + _mode = lit("GCM") if mode is None else mode + _padding = lit("DEFAULT") if padding is None else padding + _aad = lit("") if aad is None else aad + return _invoke_function_over_columns("try_aes_decrypt", input, key, _mode, _padding, _aad) @_try_remote_functions -def tuple_difference_theta_integer(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: +def sha(col: "ColumnOrName") -> Column: """ - Subtracts a Datasketches ThetaSketch from a TupleSketch with integer summaries - (elements in TupleSketch but not in ThetaSketch). + Returns a sha1 hash value as a hex string of the `col`. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with integer summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the difference TupleSketch. + col : :class:`~pyspark.sql.Column` or column name + A column that evaluates to a binary. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_integer` - :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` + :meth:`pyspark.sql.functions.sha1` + :meth:`pyspark.sql.functions.sha2` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(5, 5, 4), (1, 1, 4), (2, 2, 5), (3, 3, 1)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_difference_theta_integer(df.sketch1, "sketch2"))).show() # noqa - +-------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_difference_theta_integer(sketch1, sketch2))| - +-------------------------------------------------------------------------------+ - | 2.0| - +-------------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select(sf.sha(sf.lit("Spark"))).show() + +--------------------+ + | sha(Spark)| + +--------------------+ + |85f5955f4b27a9a4c...| + +--------------------+ """ - return _invoke_function_over_columns("tuple_difference_theta_integer", col1, col2) + return _invoke_function_over_columns("sha", col) @_try_remote_functions -def tuple_intersection_theta_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def input_file_block_length() -> Column: """ - Intersects a Datasketches TupleSketch with double summaries with a ThetaSketch. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with double summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + Returns the length of the block being read, or -1 if not available. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + .. versionadded:: 3.5.0 See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_double` - :meth:`pyspark.sql.functions.tuple_intersection_agg_double` + :meth:`pyspark.sql.functions.input_file_name` + :meth:`pyspark.sql.functions.input_file_block_start` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1.0, 1), (2, 2.0, 2), (3, 3.0, 4)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_intersection_theta_double(df.sketch1, "sketch2"))).show() # noqa - +------------------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_intersection_theta_double(sketch1, sketch2, sum))| - +------------------------------------------------------------------------------------+ - | 2.0| - +------------------------------------------------------------------------------------+ + >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") + >>> df.select(sf.input_file_block_length()).show() + +-------------------------+ + |input_file_block_length()| + +-------------------------+ + | 87| + | 87| + | 87| + | 87| + | 87| + | 87| + | 87| + | 87| + +-------------------------+ """ - fn = "tuple_intersection_theta_double" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) - else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + return _invoke_function_over_columns("input_file_block_length") @_try_remote_functions -def tuple_intersection_theta_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - mode: Optional[Union[str, Column]] = None, -) -> Column: +def input_file_block_start() -> Column: """ - Intersects a Datasketches TupleSketch with integer summaries with a ThetaSketch. - - .. versionadded:: 4.2.0 - - Parameters - ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with integer summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" + Returns the start offset of the block being read, or -1 if not available. - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the intersected TupleSketch. + .. versionadded:: 3.5.0 See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_theta_integer` - :meth:`pyspark.sql.functions.tuple_intersection_agg_integer` + :meth:`pyspark.sql.functions.input_file_name` + :meth:`pyspark.sql.functions.input_file_block_length` Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 1, 1), (2, 2, 2), (3, 3, 4)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_intersection_theta_integer(df.sketch1, "sketch2"))).show() # noqa - +--------------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_intersection_theta_integer(sketch1, sketch2, sum))| - +--------------------------------------------------------------------------------------+ - | 2.0| - +--------------------------------------------------------------------------------------+ + >>> df = spark.read.text("python/test_support/sql/ages_newlines.csv", lineSep=",") + >>> df.select(sf.input_file_block_start()).show() + +------------------------+ + |input_file_block_start()| + +------------------------+ + | 0| + | 0| + | 0| + | 0| + | 0| + | 0| + | 0| + | 0| + +------------------------+ """ - fn = "tuple_intersection_theta_integer" - if mode is None: - return _invoke_function_over_columns(fn, col1, col2) - else: - return _invoke_function_over_columns(fn, col1, col2, lit(mode)) + return _invoke_function_over_columns("input_file_block_start") @_try_remote_functions -def tuple_union_theta_double( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def reflect(*cols: "ColumnOrName") -> Column: """ - Merges a Datasketches TupleSketch with double summaries with a ThetaSketch. + Calls a method with reflection. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with double summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" - - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a Column representing literal string for the class name, + and the second element should be a Column representing literal string for the method name, + and the remaining are input arguments (Columns or column names) to the Java method. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_double` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_agg_double` - :meth:`pyspark.sql.functions.tuple_intersection_theta_double` + :meth:`pyspark.sql.functions.java_method` + :meth:`pyspark.sql.functions.try_reflect` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10.0, 3), (2, 20.0, 4)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_double("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_double(sf.tuple_union_theta_double(df.sketch1, "sketch2"))).show() # noqa - +---------------------------------------------------------------------------------+ - |tuple_sketch_estimate_double(tuple_union_theta_double(sketch1, sketch2, 12, sum))| - +---------------------------------------------------------------------------------+ - | 4.0| - +---------------------------------------------------------------------------------+ + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) + >>> df.select( + ... sf.reflect(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') + ... ).show(truncate=False) + +--------------------------------------+ + |reflect(java.util.UUID, fromString, a)| + +--------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +--------------------------------------+ """ - fn = "tuple_union_theta_double" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) + return _invoke_function_over_seq_of_columns("reflect", cols) @_try_remote_functions -def tuple_union_theta_integer( - col1: "ColumnOrName", - col2: "ColumnOrName", - lgNomEntries: Optional[Union[int, Column]] = None, - mode: Optional[Union[str, Column]] = None, -) -> Column: +def java_method(*cols: "ColumnOrName") -> Column: """ - Merges a Datasketches TupleSketch with integer summaries with a ThetaSketch. + Calls a method with reflection. - .. versionadded:: 4.2.0 + .. versionadded:: 3.5.0 Parameters ---------- - col1 : :class:`~pyspark.sql.Column` or column name - The TupleSketch column with integer summaries - col2 : :class:`~pyspark.sql.Column` or column name - The ThetaSketch column - lgNomEntries : :class:`~pyspark.sql.Column` or int, optional - The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12) - A column that evaluates to an integer. - mode : :class:`~pyspark.sql.Column` or str, optional - The summary mode: "sum" (default), "min", "max", or "alwaysone" - - Returns - ------- - :class:`~pyspark.sql.Column` - The binary representation of the merged TupleSketch. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a Column representing literal string for the class name, + and the second element should be a Column representing literal string for the method name, + and the remaining are input arguments (Columns or column names) to the Java method. See Also -------- - :meth:`pyspark.sql.functions.tuple_sketch_agg_integer` - :meth:`pyspark.sql.functions.theta_sketch_agg` - :meth:`pyspark.sql.functions.tuple_union_agg_integer` - :meth:`pyspark.sql.functions.tuple_intersection_theta_integer` + :meth:`pyspark.sql.functions.reflect` + :meth:`pyspark.sql.functions.try_reflect` Examples -------- - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(1, 10, 3), (2, 20, 4)], ["key1", "v1", "key2"]) # noqa - >>> df = df.agg( - ... sf.tuple_sketch_agg_integer("key1", "v1").alias("sketch1"), - ... sf.theta_sketch_agg("key2").alias("sketch2") - ... ) - >>> df.select(sf.tuple_sketch_estimate_integer(sf.tuple_union_theta_integer(df.sketch1, "sketch2"))).show() # noqa - +-----------------------------------------------------------------------------------+ - |tuple_sketch_estimate_integer(tuple_union_theta_integer(sketch1, sketch2, 12, sum))| - +-----------------------------------------------------------------------------------+ - | 4.0| - +-----------------------------------------------------------------------------------+ - """ - fn = "tuple_union_theta_integer" - _lgNomEntries = lit(12) if lgNomEntries is None else lit(lgNomEntries) - _mode = lit("sum") if mode is None else lit(mode) - - return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) - + Example 1: Reflecting a method call with a column argument -# ---------------------- Geospatial ST Functions ---------------------- + >>> import pyspark.sql.functions as sf + >>> spark.range(1).select( + ... sf.java_method( + ... sf.lit("java.util.UUID"), + ... sf.lit("fromString"), + ... sf.lit("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2") + ... ) + ... ).show(truncate=False) + +-----------------------------------------------------------------------------+ + |java_method(java.util.UUID, fromString, a5cf6c42-0c85-418f-af6c-3e4e5b1328f2)| + +-----------------------------------------------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +-----------------------------------------------------------------------------+ + Example 2: Reflecting a method call with a column name argument -def _ensure_column_or_name(arg: Optional[Any]) -> "ColumnOrName": - if not isinstance(arg, (Column, str)): - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "Column or str", - "arg_name": "arg", - "arg_type": type(arg).__name__, - }, - ) - return arg + >>> import pyspark.sql.functions as sf + >>> df = spark.createDataFrame([('a5cf6c42-0c85-418f-af6c-3e4e5b1328f2',)], ['a']) + >>> df.select( + ... sf.java_method(sf.lit('java.util.UUID'), sf.lit('fromString'), 'a') + ... ).show(truncate=False) + +------------------------------------------+ + |java_method(java.util.UUID, fromString, a)| + +------------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +------------------------------------------+ + """ + return _invoke_function_over_seq_of_columns("java_method", cols) @_try_remote_functions -def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None) -> Column: - """Returns the input GEOGRAPHY or GEOMETRY value in WKB format. +def try_reflect(*cols: "ColumnOrName") -> Column: + """ + This is a special version of `reflect` that performs the same operation, but returns a NULL + value instead of raising an error if the invoke method thrown exception. - .. versionadded:: 4.1.0 - .. versionchanged:: 4.2.0 - Added the optional `endianness` parameter. + .. versionadded:: 4.0.0 Parameters ---------- - geo : :class:`~pyspark.sql.Column` or str - A geospatial value, either a GEOGRAPHY or a GEOMETRY. - endianness : :class:`~pyspark.sql.Column` or str, optional - The optional endianness of the output WKB, 'NDR' for little-endian (default) or 'XDR' for - big-endian. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a Column representing literal string for the class name, + and the second element should be a Column representing literal string for the method name, + and the remaining are input arguments (Columns or column names) to the Java method. - Examples + See Also -------- + :meth:`pyspark.sql.functions.reflect` + :meth:`pyspark.sql.functions.java_method` - Example 1: Getting WKB from GEOGRAPHY. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] + Examples + -------- + Example 1: Reflecting a method call with arguments - Example 2: Getting WKB from GEOMETRY. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] + >>> df = spark.createDataFrame([("a5cf6c42-0c85-418f-af6c-3e4e5b1328f2",)], ["a"]) + >>> df.select( + ... sf.try_reflect(sf.lit("java.util.UUID"), sf.lit("fromString"), "a") + ... ).show(truncate=False) + +------------------------------------------+ + |try_reflect(java.util.UUID, fromString, a)| + +------------------------------------------+ + |a5cf6c42-0c85-418f-af6c-3e4e5b1328f2 | + +------------------------------------------+ - Example 3: Getting WKB (little-endian) from GEOGRAPHY. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb'), 'NDR'))).collect() - [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] + Example 2: Exception in the reflection call, resulting in null - Example 4: Getting WKB (big-endian) from GEOMETRY. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb'), 'XDR'))).collect() - [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), XDR))='00000000013FF00000000000004000000000000000')] + >>> spark.range(1).select( + ... sf.try_reflect(sf.lit("scala.Predef"), sf.lit("require"), sf.lit(False)) + ... ).show(truncate=False) + +-----------------------------------------+ + |try_reflect(scala.Predef, require, false)| + +-----------------------------------------+ + |NULL | + +-----------------------------------------+ """ - if endianness is None: - return _invoke_function_over_columns("st_asbinary", geo) - else: - _endianness = lit(endianness) if isinstance(endianness, str) else endianness - return _invoke_function_over_columns("st_asbinary", geo, _endianness) + return _invoke_function_over_seq_of_columns("try_reflect", cols) @_try_remote_functions -def st_geogfromwkb(wkb: "ColumnOrName") -> Column: - """Parses the input WKB description and returns the corresponding GEOGRAPHY value. - - .. versionadded:: 4.1.0 +def version() -> Column: + """ + Returns the Spark version. The string contains 2 fields, the first being a release version + and the second being a git revision. - Parameters - ---------- - wkb : :class:`~pyspark.sql.Column` or str - A BINARY value in WKB format, representing a GEOGRAPHY value. - A column that evaluates to a binary. + .. versionadded:: 3.5.0 Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geogfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geogfromwkb(wkb), NDR))='0101000000000000000000F03F0000000000000040')] + >>> spark.range(1).select(sf.version()).show(truncate=False) # doctest: +SKIP + +----------------------------------------------+ + |version() | + +----------------------------------------------+ + |4.0.0 4f8d1f575e99aeef8990c63a9614af0fc5479330| + +----------------------------------------------+ """ - return _invoke_function_over_columns("st_geogfromwkb", wkb) + return _invoke_function_over_columns("version") @_try_remote_functions -def st_geomfromwkb( - wkb: "ColumnOrName", srid: Optional[Union["ColumnOrName", int]] = None -) -> Column: - """Parses the input WKB description and returns the corresponding GEOMETRY value. +def typeof(col: "ColumnOrName") -> Column: + """ + Return DDL-formatted type string for the data type of the input. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - wkb : :class:`~pyspark.sql.Column` or str - A BINARY value in WKB format, representing a GEOMETRY value. - A column that evaluates to a binary. - srid : :class:`~pyspark.sql.Column` or int, optional - The optional SRID value of the geometry. Default is 0. - A column that evaluates to an integer. + col : :class:`~pyspark.sql.Column` or column name Examples -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.hex(sf.st_asbinary(sf.st_geomfromwkb('wkb')))).collect() - [Row(hex(st_asbinary(st_geomfromwkb(wkb, 0), NDR))='0101000000000000000000F03F0000000000000040')] + >>> df = spark.createDataFrame([(True, 1, 1.0, 'xyz',)], ['a', 'b', 'c', 'd']) + >>> df.select(sf.typeof(df.a), sf.typeof(df.b), sf.typeof('c'), sf.typeof('d')).show() + +---------+---------+---------+---------+ + |typeof(a)|typeof(b)|typeof(c)|typeof(d)| + +---------+---------+---------+---------+ + | boolean| bigint| double| string| + +---------+---------+---------+---------+ """ - if srid is None: - return _invoke_function_over_columns("st_geomfromwkb", wkb) - else: - srid = _enum_to_value(srid) - srid = lit(srid) if isinstance(srid, int) else srid - return _invoke_function_over_columns("st_geomfromwkb", wkb, srid) + return _invoke_function_over_columns("typeof", col) @_try_remote_functions -def st_setsrid(geo: "ColumnOrName", srid: Union["ColumnOrName", int]) -> Column: - """Returns a new GEOGRAPHY or GEOMETRY value whose SRID is the specified SRID value. +def stack(*cols: "ColumnOrName") -> Column: + """ + Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default + unless specified otherwise. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - geo : :class:`~pyspark.sql.Column` or str - A geospatial value, either a GEOGRAPHY or a GEOMETRY. - srid : :class:`~pyspark.sql.Column` or int - An INTEGER representing the new SRID of the geospatial value. + cols : :class:`~pyspark.sql.Column` or column name + the first element should be a literal int for the number of rows to be separated, + and the remaining are input elements to be separated. Examples -------- - - Example 1: Setting the SRID on GEOGRAPHY with SRID from another column. >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'), 4326)], ['wkb', 'srid']) # noqa - >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geogfromwkb('wkb'), 'srid'))).collect() - [Row(st_srid(st_setsrid(st_geogfromwkb(wkb), srid))=4326)] + >>> df = spark.createDataFrame([(1, 2, 3)], ['a', 'b', 'c']) + >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c')).show() + +---+---+---+----+----+ + | a| b| c|col0|col1| + +---+---+---+----+----+ + | 1| 2| 3| 1| 2| + | 1| 2| 3| 3|NULL| + +---+---+---+----+----+ - Example 2: Setting the SRID on GEOMETRY with SRID as an integer literal. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.st_srid(sf.st_setsrid(sf.st_geomfromwkb('wkb'), 4326))).collect() - [Row(st_srid(st_setsrid(st_geomfromwkb(wkb, 0), 4326))=4326)] + >>> df.select('*', sf.stack(sf.lit(2), df.a, df.b, 'c').alias('x', 'y')).show() + +---+---+---+---+----+ + | a| b| c| x| y| + +---+---+---+---+----+ + | 1| 2| 3| 1| 2| + | 1| 2| 3| 3|NULL| + +---+---+---+---+----+ + + >>> df.select('*', sf.stack(sf.lit(3), df.a, df.b, 'c')).show() + +---+---+---+----+ + | a| b| c|col0| + +---+---+---+----+ + | 1| 2| 3| 1| + | 1| 2| 3| 2| + | 1| 2| 3| 3| + +---+---+---+----+ + + >>> df.select('*', sf.stack(sf.lit(4), df.a, df.b, 'c')).show() + +---+---+---+----+ + | a| b| c|col0| + +---+---+---+----+ + | 1| 2| 3| 1| + | 1| 2| 3| 2| + | 1| 2| 3| 3| + | 1| 2| 3|NULL| + +---+---+---+----+ """ - srid = _enum_to_value(srid) - srid = lit(srid) if isinstance(srid, int) else srid - return _invoke_function_over_columns("st_setsrid", geo, srid) + return _invoke_function_over_seq_of_columns("stack", cols) @_try_remote_functions -def st_srid(geo: "ColumnOrName") -> Column: - """Returns the SRID of the input GEOGRAPHY or GEOMETRY value. +def bitmap_bit_position(col: "ColumnOrName") -> Column: + """ + Returns the bit position for the given input column. - .. versionadded:: 4.1.0 + .. versionadded:: 3.5.0 Parameters ---------- - geo : :class:`~pyspark.sql.Column` or str - A geospatial value, either a GEOGRAPHY or a GEOMETRY. + col : :class:`~pyspark.sql.Column` or column name + The input column. + A column that evaluates to a long. - Examples + See Also -------- + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` - Example 1: Getting the SRID of GEOGRAPHY. - >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.st_srid(sf.st_geogfromwkb('wkb'))).collect() - [Row(st_srid(st_geogfromwkb(wkb))=4326)] - - Example 2: Getting the SRID of GEOMETRY. + Examples + -------- >>> from pyspark.sql import functions as sf - >>> df = spark.createDataFrame([(bytes.fromhex('0101000000000000000000F03F0000000000000040'),)], ['wkb']) # noqa - >>> df.select(sf.st_srid(sf.st_geomfromwkb('wkb'))).collect() - [Row(st_srid(st_geomfromwkb(wkb, 0))=0)] + >>> df = spark.createDataFrame([(123,)], ['a']) + >>> df.select('*', sf.bitmap_bit_position('a')).show() + +---+----------------------+ + | a|bitmap_bit_position(a)| + +---+----------------------+ + |123| 122| + +---+----------------------+ """ - return _invoke_function_over_columns("st_srid", geo) - - -# ---------------------- Vector Functions ---------------------- + return _invoke_function_over_columns("bitmap_bit_position", col) @_try_remote_functions -def vector_cosine_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Returns the cosine similarity between two float vectors. - The vectors must have the same dimension. +def bitmap_bucket_number(col: "ColumnOrName") -> Column: + """ + Returns the bucket number for the given input column. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - first vector column. - right : :class:`~pyspark.sql.Column` or column name - second vector column. + col : :class:`~pyspark.sql.Column` or column name + The input column. + A column that evaluates to a long. - Returns - ------- - :class:`~pyspark.sql.Column` - cosine similarity as a float value. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) - >>> df.select(sf.vector_cosine_similarity('a', 'b')).first()[0] - 0.974631... + >>> df = spark.createDataFrame([(123,)], ['a']) + >>> df.select('*', sf.bitmap_bucket_number('a')).show() + +---+-----------------------+ + | a|bitmap_bucket_number(a)| + +---+-----------------------+ + |123| 1| + +---+-----------------------+ """ - return _invoke_function_over_columns("vector_cosine_similarity", left, right) + return _invoke_function_over_columns("bitmap_bucket_number", col) @_try_remote_functions -def vector_inner_product(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Returns the inner product (dot product) between two float vectors. - The vectors must have the same dimension. +def bitmap_construct_agg(col: "ColumnOrName") -> Column: + """ + Returns a bitmap with the positions of the bits set from all the values from the input column. + The input column will most likely be bitmap_bit_position(). - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - first vector column. - right : :class:`~pyspark.sql.Column` or column name - second vector column. + col : :class:`~pyspark.sql.Column` or column name + The input column will most likely be bitmap_bit_position(). + A column that evaluates to a long. - Returns - ------- - :class:`~pyspark.sql.Column` - inner product as a float value. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` + :meth:`pyspark.sql.functions.bitmap_and_agg` Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) - >>> df.select(sf.vector_inner_product('a', 'b')).first()[0] - 32.0 + >>> df = spark.createDataFrame([(1,),(2,),(3,)], ["a"]) + >>> df.select( + ... sf.bitmap_construct_agg(sf.bitmap_bit_position('a')) + ... ).show() + +--------------------------------------------+ + |bitmap_construct_agg(bitmap_bit_position(a))| + +--------------------------------------------+ + | [07 00 00 00 00 0...| + +--------------------------------------------+ """ - return _invoke_function_over_columns("vector_inner_product", left, right) + return _invoke_function_over_columns("bitmap_construct_agg", col) @_try_remote_functions -def vector_l2_distance(left: "ColumnOrName", right: "ColumnOrName") -> Column: - """Returns the Euclidean (L2) distance between two float vectors. - The vectors must have the same dimension. +def bitmap_count(col: "ColumnOrName") -> Column: + """ + Returns the number of set bits in the input bitmap. - .. versionadded:: 4.3.0 + .. versionadded:: 3.5.0 Parameters ---------- - left : :class:`~pyspark.sql.Column` or column name - first vector column. - right : :class:`~pyspark.sql.Column` or column name - second vector column. + col : :class:`~pyspark.sql.Column` or column name + The input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - L2 distance as a float value. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_or_agg` Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) - >>> df.select(sf.vector_l2_distance('a', 'b')).first()[0] - 5.196152... + >>> df = spark.createDataFrame([("FFFF",)], ["a"]) + >>> df.select(sf.bitmap_count(sf.to_binary(df.a, sf.lit("hex")))).show() + +-------------------------------+ + |bitmap_count(to_binary(a, hex))| + +-------------------------------+ + | 16| + +-------------------------------+ """ - return _invoke_function_over_columns("vector_l2_distance", left, right) + return _invoke_function_over_columns("bitmap_count", col) @_try_remote_functions -def vector_norm(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: - """Returns the Lp norm of a float vector using the specified degree. - Degree defaults to 2.0 (Euclidean norm) if unspecified. +def bitmap_and(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """ + Returns a bitmap that is the bitwise AND of two input bitmaps. - .. versionadded:: 4.3.0 + .. versionadded:: 4.4.0 Parameters ---------- - vector : :class:`~pyspark.sql.Column` or column name - input vector column. - degree : :class:`~pyspark.sql.Column` or column name, optional - norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). - Defaults to 2.0. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - the Lp norm as a float value. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_andnot` + :meth:`pyspark.sql.functions.bitmap_or` + :meth:`pyspark.sql.functions.bitmap_xor` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) - >>> df.select(sf.vector_norm('v', sf.lit(2.0).cast('float'))).first()[0] - 5.0 + >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) + >>> df.select(sf.bitmap_and( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[70 00 00 00 00 0...| + +--------------------+ """ - if degree is None: - return _invoke_function_over_columns("vector_norm", vector) - else: - return _invoke_function_over_columns("vector_norm", vector, degree) + return _invoke_function_over_columns("bitmap_and", left, right) @_try_remote_functions -def vector_normalize(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: - """Normalizes a float vector to unit length using the specified norm degree. - Degree defaults to 2.0 (Euclidean norm) if unspecified. +def bitmap_or(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """ + Returns a bitmap that is the bitwise OR of two input bitmaps. - .. versionadded:: 4.3.0 + .. versionadded:: 4.4.0 Parameters ---------- - vector : :class:`~pyspark.sql.Column` or column name - input vector column. - degree : :class:`~pyspark.sql.Column` or column name, optional - norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). - Defaults to 2.0. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - the normalized vector as an array of floats. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_and` + :meth:`pyspark.sql.functions.bitmap_andnot` + :meth:`pyspark.sql.functions.bitmap_xor` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) - >>> df.select(sf.vector_normalize('v', sf.lit(2.0).cast('float'))).first()[0] - [0.6..., 0.8...] + >>> df = spark.createDataFrame([("10", "20")], ["left", "right"]) + >>> df.select(sf.bitmap_or( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[30 00 00 00 00 0...| + +--------------------+ """ - if degree is None: - return _invoke_function_over_columns("vector_normalize", vector) - else: - return _invoke_function_over_columns("vector_normalize", vector, degree) + return _invoke_function_over_columns("bitmap_or", left, right) @_try_remote_functions -def vector_avg(col: "ColumnOrName") -> Column: - """Aggregate function: returns the element-wise mean of float vectors in a group. - All vectors must have the same dimension. +def bitmap_andnot(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """ + Returns a bitmap that is the bitwise AND NOT of two input bitmaps. - .. versionadded:: 4.3.0 + .. versionadded:: 4.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input vector column. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - the element-wise average vector as an array of floats. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_and` + :meth:`pyspark.sql.functions.bitmap_or` + :meth:`pyspark.sql.functions.bitmap_xor` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) - >>> df.select(sf.vector_avg('v')).first()[0] - [2.0, 3.0] + >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) + >>> df.select(sf.bitmap_andnot( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[80 00 00 00 00 0...| + +--------------------+ """ - return _invoke_function_over_columns("vector_avg", col) + return _invoke_function_over_columns("bitmap_andnot", left, right) @_try_remote_functions -def vector_sum(col: "ColumnOrName") -> Column: - """Aggregate function: returns the element-wise sum of float vectors in a group. - All vectors must have the same dimension. +def bitmap_xor(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """ + Returns a bitmap that is the bitwise XOR of two input bitmaps. - .. versionadded:: 4.3.0 + .. versionadded:: 4.4.0 Parameters ---------- - col : :class:`~pyspark.sql.Column` or column name - input vector column. + left : :class:`~pyspark.sql.Column` or column name + The left input bitmap. + right : :class:`~pyspark.sql.Column` or column name + The right input bitmap. - Returns - ------- - :class:`~pyspark.sql.Column` - the element-wise sum vector as an array of floats. + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_and` + :meth:`pyspark.sql.functions.bitmap_andnot` + :meth:`pyspark.sql.functions.bitmap_or` + + Notes + ----- + Inputs use Spark's Binary bitmap representation, not a RoaringBitmap serialization. Each + input may contain 0 to 4096 bytes; missing bytes are treated as zero. The result is always a + 4096-byte Binary value. NULL input returns NULL, and inputs longer than 4096 bytes raise + ``BITMAP_INPUT_TOO_LARGE``. Both inputs must use the same bit-position mapping. If they were + constructed by grouping ``bitmap_bit_position`` values by ``bitmap_bucket_number``, they must + represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + function combines two bitmaps from the same row; use ``bitmap_*_agg`` to combine bitmaps across + rows. Examples -------- >>> from pyspark.sql import functions as sf - >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField - >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) - >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) - >>> df.select(sf.vector_sum('v')).first()[0] - [4.0, 6.0] + >>> df = spark.createDataFrame([("F0", "70")], ["left", "right"]) + >>> df.select(sf.bitmap_xor( + ... sf.to_binary("left", sf.lit("hex")), + ... sf.to_binary("right", sf.lit("hex"))).alias("bitmap")).show() + +--------------------+ + | bitmap| + +--------------------+ + |[80 00 00 00 00 0...| + +--------------------+ """ - return _invoke_function_over_columns("vector_sum", col) - - -# ---------------------- UDF, UDTF and UDT ---------------------- + return _invoke_function_over_columns("bitmap_xor", left, right) @_try_remote_functions -def call_udf(udfName: str, *cols: "ColumnOrName") -> Column: +def bitmap_or_agg(col: "ColumnOrName") -> Column: """ - Call a user-defined function. + Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. + The input column should be bitmaps created from bitmap_construct_agg(). - .. versionadded:: 3.4.0 + .. versionadded:: 3.5.0 + + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_and_agg` Parameters ---------- - udfName : str - name of the user defined function (UDF) - cols : :class:`~pyspark.sql.Column` or str - column names or :class:`~pyspark.sql.Column`\\s to be used in the UDF - - Returns - ------- - :class:`~pyspark.sql.Column` - result of executed udf. + col : :class:`~pyspark.sql.Column` or column name + The input column should be bitmaps created from bitmap_construct_agg(). Examples -------- - >>> from pyspark.sql.functions import call_udf, col - >>> from pyspark.sql.types import IntegerType, StringType - >>> df = spark.createDataFrame([(1, "a"),(2, "b"), (3, "c")],["id", "name"]) - >>> _ = spark.udf.register("intX2", lambda i: i * 2, IntegerType()) - >>> df.select(call_udf("intX2", "id")).show() - +---------+ - |intX2(id)| - +---------+ - | 2| - | 4| - | 6| - +---------+ - >>> _ = spark.udf.register("strX2", lambda s: s * 2, StringType()) - >>> df.select(call_udf("strX2", col("name"))).show() - +-----------+ - |strX2(name)| - +-----------+ - | aa| - | bb| - | cc| - +-----------+ + >>> from pyspark.sql import functions as sf + >>> df = spark.createDataFrame([("10",),("20",),("40",)], ["a"]) + >>> df.select(sf.bitmap_or_agg(sf.to_binary(df.a, sf.lit("hex")))).show() + +--------------------------------+ + |bitmap_or_agg(to_binary(a, hex))| + +--------------------------------+ + | [70 00 00 00 00 0...| + +--------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column, _to_seq - - sc = _get_active_spark_context() - return _invoke_function("call_udf", udfName, _to_seq(sc, cols, _to_java_column)) + return _invoke_function_over_columns("bitmap_or_agg", col) @_try_remote_functions -def unwrap_udt(col: "ColumnOrName") -> Column: +def bitmap_and_agg(col: "ColumnOrName") -> Column: """ - Unwrap UDT data type column into its underlying type. + Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. + The input column should be bitmaps created from bitmap_construct_agg(). - .. versionadded:: 3.4.0 + .. versionadded:: 4.1.0 + + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - - Returns - ------- - :class:`~pyspark.sql.Column` - The underlying representation. - - See Also - -------- - :meth:`pyspark.sql.functions.wrap_udt` + The input column should be bitmaps created from bitmap_construct_agg(). Examples -------- - Example 1: Unwrap ML-specific UDT - VectorUDT - - >>> from pyspark.sql import functions as sf - >>> from pyspark.ml.linalg import Vectors - >>> vec1 = Vectors.dense(1, 2, 3) - >>> vec2 = Vectors.sparse(4, {1: 1.0, 3: 5.5}) - >>> df = spark.createDataFrame([(vec1,), (vec2,)], ["vec"]) - >>> df.select(sf.unwrap_udt("vec")).printSchema() - root - |-- unwrap_udt(vec): struct (nullable = true) - | |-- type: byte (nullable = false) - | |-- size: integer (nullable = true) - | |-- indices: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- values: array (nullable = true) - | | |-- element: double (containsNull = false) - - Example 2: Unwrap ML-specific UDT - MatrixUDT - >>> from pyspark.sql import functions as sf - >>> from pyspark.ml.linalg import Matrices - >>> mat1 = Matrices.dense(2, 2, range(4)) - >>> mat2 = Matrices.sparse(2, 2, [0, 2, 3], [0, 1, 1], [2, 3, 4]) - >>> df = spark.createDataFrame([(mat1,), (mat2,)], ["mat"]) - >>> df.select(sf.unwrap_udt("mat")).printSchema() - root - |-- unwrap_udt(mat): struct (nullable = true) - | |-- type: byte (nullable = false) - | |-- numRows: integer (nullable = false) - | |-- numCols: integer (nullable = false) - | |-- colPtrs: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- rowIndices: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- values: array (nullable = true) - | | |-- element: double (containsNull = false) - | |-- isTransposed: boolean (nullable = false) + >>> df = spark.createDataFrame([("F0",),("70",),("30",)], ["a"]) + >>> df.select(sf.bitmap_and_agg(sf.to_binary(df.a, sf.lit("hex")))).show() + +---------------------------------+ + |bitmap_and_agg(to_binary(a, hex))| + +---------------------------------+ + | [30 00 00 00 00 0...| + +---------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column - - return _invoke_function("unwrap_udt", _to_java_column(col)) + return _invoke_function_over_columns("bitmap_and_agg", col) @_try_remote_functions -def wrap_udt(col: "ColumnOrName", udt: "Union[UserDefinedType, Column]") -> Column: +def bitmap_xor_agg(col: "ColumnOrName") -> Column: """ - Wrap a column as a user-defined type. + Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. + The input column should be bitmaps created from bitmap_construct_agg(). .. versionadded:: 4.4.0 + See Also + -------- + :meth:`pyspark.sql.functions.bitmap_bit_position` + :meth:`pyspark.sql.functions.bitmap_bucket_number` + :meth:`pyspark.sql.functions.bitmap_construct_agg` + :meth:`pyspark.sql.functions.bitmap_count` + :meth:`pyspark.sql.functions.bitmap_or_agg` + :meth:`pyspark.sql.functions.bitmap_and_agg` + Parameters ---------- col : :class:`~pyspark.sql.Column` or column name - The column to wrap. The column data type must match the UDT's underlying SQL type. - udt : :class:`~pyspark.sql.types.UserDefinedType` or :class:`~pyspark.sql.Column` - The target user-defined type, or a constant string column containing its JSON - representation. - - Returns - ------- - :class:`~pyspark.sql.Column` - A column of the target user-defined type. - - See Also - -------- - :meth:`pyspark.sql.functions.unwrap_udt` + The input column should be bitmaps created from bitmap_construct_agg(). Examples -------- - Example 1: Wrapping a vector struct as VectorUDT - - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Row - >>> from pyspark.sql.types import StructField, StructType - >>> from pyspark.ml.linalg import VectorUDT - >>> vector_schema = StructType([StructField("vec", VectorUDT.sqlType(), True)]) - >>> df = spark.createDataFrame( - ... [(Row(type=1, size=None, indices=None, values=[1.0, 2.0, 3.0]),)], - ... vector_schema) - >>> df.select("*", sf.wrap_udt("vec", VectorUDT())).show() - +--------------------+...+ - | vec|wrap_udt(vec...| - +--------------------+...+ - |{1, NULL, NULL, [...|...[1.0,2.0,3.0]| - +--------------------+...+ - >>> row = df.select(sf.wrap_udt("vec", VectorUDT())).first() - >>> type(row[0]) - - - Example 2: Wrapping a matrix struct as MatrixUDT - >>> from pyspark.sql import functions as sf - >>> from pyspark.sql import Row - >>> from pyspark.sql.types import StructField, StructType - >>> from pyspark.mllib.linalg import MatrixUDT - >>> matrix_schema = StructType([StructField("mat", MatrixUDT.sqlType(), True)]) - >>> df = spark.createDataFrame( - ... [( - ... Row( - ... type=1, - ... numRows=2, - ... numCols=2, - ... colPtrs=None, - ... rowIndices=None, - ... values=[1.0, 2.0, 3.0, 4.0], - ... isTransposed=False), - ... )], - ... matrix_schema) - >>> df.select("*", sf.wrap_udt("mat", MatrixUDT())).printSchema() - root - |-- mat: struct (nullable = true) - | |-- type: byte (nullable = false) - | |-- numRows: integer (nullable = false) - | |-- numCols: integer (nullable = false) - | |-- colPtrs: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- rowIndices: array (nullable = true) - | | |-- element: integer (containsNull = false) - | |-- values: array (nullable = true) - | | |-- element: double (containsNull = false) - | |-- isTransposed: boolean (nullable = false) - |-- wrap_udt(mat...: matrix... (nullable = true) - >>> row = df.select(sf.wrap_udt("mat", MatrixUDT())).first() - >>> type(row[0]) - + >>> df = spark.createDataFrame([("10",), ("30",), ("40",)], ["a"]) + >>> df.select(sf.bitmap_xor_agg(sf.to_binary(df.a, sf.lit("hex")))).show() + +---------------------------------+ + |bitmap_xor_agg(to_binary(a, hex))| + +---------------------------------+ + | [60 00 00 00 00 0...| + +---------------------------------+ """ - from pyspark.sql.classic.column import _to_java_column + return _invoke_function_over_columns("bitmap_xor_agg", col) - if isinstance(udt, _UserDefinedType): - udt_col = lit(udt.json()) - elif isinstance(udt, Column): - udt_col = udt - else: - raise PySparkTypeError( - errorClass="NOT_EXPECTED_TYPE", - messageParameters={ - "expected_type": "UserDefinedType or Column", - "arg_name": "udt", - "arg_type": type(udt).__name__, - }, - ) - return _invoke_function("wrap_udt", _to_java_column(col), _to_java_column(udt_col)) + +# ---------------------------- User Defined Function ---------------------------------- def udaf(agg: "Aggregator") -> "UserDefinedFunctionLike": @@ -34304,6 +34041,230 @@ def arrow_udtf( return _create_pyarrow_udtf(cls=cls, returnType=returnType) +# ---------------------- Vector Functions ---------------------- + + +@_try_remote_functions +def vector_cosine_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Returns the cosine similarity between two float vectors. + The vectors must have the same dimension. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + left : :class:`~pyspark.sql.Column` or column name + first vector column. + right : :class:`~pyspark.sql.Column` or column name + second vector column. + + Returns + ------- + :class:`~pyspark.sql.Column` + cosine similarity as a float value. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) + >>> df.select(sf.vector_cosine_similarity('a', 'b')).first()[0] + 0.974631... + """ + return _invoke_function_over_columns("vector_cosine_similarity", left, right) + + +@_try_remote_functions +def vector_inner_product(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Returns the inner product (dot product) between two float vectors. + The vectors must have the same dimension. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + left : :class:`~pyspark.sql.Column` or column name + first vector column. + right : :class:`~pyspark.sql.Column` or column name + second vector column. + + Returns + ------- + :class:`~pyspark.sql.Column` + inner product as a float value. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) + >>> df.select(sf.vector_inner_product('a', 'b')).first()[0] + 32.0 + """ + return _invoke_function_over_columns("vector_inner_product", left, right) + + +@_try_remote_functions +def vector_l2_distance(left: "ColumnOrName", right: "ColumnOrName") -> Column: + """Returns the Euclidean (L2) distance between two float vectors. + The vectors must have the same dimension. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + left : :class:`~pyspark.sql.Column` or column name + first vector column. + right : :class:`~pyspark.sql.Column` or column name + second vector column. + + Returns + ------- + :class:`~pyspark.sql.Column` + L2 distance as a float value. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('a', ArrayType(FloatType())), StructField('b', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0, 3.0], [4.0, 5.0, 6.0])], schema) + >>> df.select(sf.vector_l2_distance('a', 'b')).first()[0] + 5.196152... + """ + return _invoke_function_over_columns("vector_l2_distance", left, right) + + +@_try_remote_functions +def vector_norm(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: + """Returns the Lp norm of a float vector using the specified degree. + Degree defaults to 2.0 (Euclidean norm) if unspecified. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + vector : :class:`~pyspark.sql.Column` or column name + input vector column. + degree : :class:`~pyspark.sql.Column` or column name, optional + norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). + Defaults to 2.0. + + Returns + ------- + :class:`~pyspark.sql.Column` + the Lp norm as a float value. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) + >>> df.select(sf.vector_norm('v', sf.lit(2.0).cast('float'))).first()[0] + 5.0 + """ + if degree is None: + return _invoke_function_over_columns("vector_norm", vector) + else: + return _invoke_function_over_columns("vector_norm", vector, degree) + + +@_try_remote_functions +def vector_normalize(vector: "ColumnOrName", degree: Optional["ColumnOrName"] = None) -> Column: + """Normalizes a float vector to unit length using the specified norm degree. + Degree defaults to 2.0 (Euclidean norm) if unspecified. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + vector : :class:`~pyspark.sql.Column` or column name + input vector column. + degree : :class:`~pyspark.sql.Column` or column name, optional + norm degree (1.0 for L1, 2.0 for L2, float('inf') for infinity norm). + Defaults to 2.0. + + Returns + ------- + :class:`~pyspark.sql.Column` + the normalized vector as an array of floats. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([3.0, 4.0],)], schema) + >>> df.select(sf.vector_normalize('v', sf.lit(2.0).cast('float'))).first()[0] + [0.6..., 0.8...] + """ + if degree is None: + return _invoke_function_over_columns("vector_normalize", vector) + else: + return _invoke_function_over_columns("vector_normalize", vector, degree) + + +@_try_remote_functions +def vector_avg(col: "ColumnOrName") -> Column: + """Aggregate function: returns the element-wise mean of float vectors in a group. + All vectors must have the same dimension. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input vector column. + + Returns + ------- + :class:`~pyspark.sql.Column` + the element-wise average vector as an array of floats. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) + >>> df.select(sf.vector_avg('v')).first()[0] + [2.0, 3.0] + """ + return _invoke_function_over_columns("vector_avg", col) + + +@_try_remote_functions +def vector_sum(col: "ColumnOrName") -> Column: + """Aggregate function: returns the element-wise sum of float vectors in a group. + All vectors must have the same dimension. + + .. versionadded:: 4.3.0 + + Parameters + ---------- + col : :class:`~pyspark.sql.Column` or column name + input vector column. + + Returns + ------- + :class:`~pyspark.sql.Column` + the element-wise sum vector as an array of floats. + + Examples + -------- + >>> from pyspark.sql import functions as sf + >>> from pyspark.sql.types import ArrayType, FloatType, StructType, StructField + >>> schema = StructType([StructField('v', ArrayType(FloatType()))]) + >>> df = spark.createDataFrame([([1.0, 2.0],), ([3.0, 4.0],)], schema) + >>> df.select(sf.vector_sum('v')).first()[0] + [4.0, 6.0] + """ + return _invoke_function_over_columns("vector_sum", col) + + def _test() -> None: import doctest diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index 4ea5b3ef0be0c..efa69f4154ce6 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -53,34 +53,34 @@ import org.apache.spark.util.SparkClassUtils * only `Column` but also other types such as a native string. The other variants currently exist * for historical reasons. * - * @groupname normal_funcs Normal Functions - * @groupname conditional_funcs Conditional Functions - * @groupname predicate_funcs Predicate Functions - * @groupname sort_funcs Sort Functions - * @groupname math_funcs Mathematical Functions - * @groupname string_funcs String Functions - * @groupname bitwise_funcs Bitwise Functions - * @groupname datetime_funcs Date and Timestamp Functions - * @groupname hash_funcs Hash Functions - * @groupname collection_funcs Collection Functions - * @groupname array_funcs Array Functions - * @groupname struct_funcs Struct Functions - * @groupname map_funcs Map Functions - * @groupname agg_funcs Aggregate Functions - * @groupname window_funcs Window Functions - * @groupname generator_funcs Generator Functions - * @groupname partition_transforms Partition Transformation Functions - * @groupname csv_funcs CSV Functions - * @groupname json_funcs JSON Functions - * @groupname variant_funcs VARIANT Functions - * @groupname xml_funcs XML Functions - * @groupname url_funcs URL Functions - * @groupname misc_funcs Misc Functions - * @groupname sketch_funcs Datasketch Functions - * @groupname st_funcs Geospatial ST Functions - * @groupname vector_funcs Vector Functions + * @groupname normal_funcs Normal functions + * @groupname conditional_funcs Conditional functions + * @groupname predicate_funcs Predicate functions + * @groupname sort_funcs Sort functions + * @groupname math_funcs Mathematical functions + * @groupname string_funcs String functions + * @groupname bitwise_funcs Bitwise functions + * @groupname datetime_funcs Date and Timestamp functions + * @groupname hash_funcs Hash functions + * @groupname collection_funcs Collection functions + * @groupname array_funcs Array functions + * @groupname struct_funcs Struct functions + * @groupname map_funcs Map functions + * @groupname agg_funcs Aggregate functions + * @groupname window_funcs Window functions + * @groupname generator_funcs Generator functions + * @groupname partition_transforms Partition transform functions + * @groupname csv_funcs CSV functions + * @groupname json_funcs JSON functions + * @groupname variant_funcs VARIANT functions + * @groupname xml_funcs XML functions + * @groupname url_funcs URL functions + * @groupname misc_funcs Misc functions + * @groupname sketch_funcs Datasketch functions + * @groupname st_funcs ST geospatial functions + * @groupname vector_funcs Vector functions * @groupname udf_funcs UDF, UDAF and UDT - * @groupname Ungrouped Support Functions for DataFrames + * @groupname Ungrouped Support functions for DataFrames * @since 1.3.0 */ @Stable @@ -88,9 +88,8 @@ import org.apache.spark.util.SparkClassUtils object functions { // scalastyle:on - ////////////////////////////////////////////////////////////////////////////////////////////// - // Normal Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + // Function groups are defined by the @group tags above each function and the corresponding + // @groupname declarations. Section headings in this implementation file are navigation aids. /** * Returns a [[Column]] based on the given column name. @@ -175,4192 +174,4661 @@ object functions { } } + ////////////////////////////////////////////////////////////////////////////////////////////// + // Sort functions + ////////////////////////////////////////////////////////////////////////////////////////////// + /** - * Marks a DataFrame as small enough for use in broadcast joins. - * - * The following example marks the right DataFrame for broadcast hash join using `joinKey`. + * Returns a sort expression based on ascending order of the column. * {{{ - * // left and right are DataFrames - * left.join(broadcast(right), "joinKey") + * df.sort(asc("dept"), desc("age")) * }}} * - * @group normal_funcs - * @since 1.5.0 + * @group sort_funcs + * @since 1.3.0 */ - def broadcast[U](df: Dataset[U]): df.type = { - df.hint("broadcast").asInstanceOf[df.type] - } + def asc(columnName: String): Column = Column(columnName).asc /** - * Parses the expression string into the column that it represents, similar to - * [[Dataset#selectExpr]]. + * Returns a sort expression based on ascending order of the column, and null values return + * before non-null values. * {{{ - * // get the number of words of each length - * df.groupBy(expr("length(word)")).count() + * df.sort(asc_nulls_first("dept"), desc("age")) * }}} * - * @group normal_funcs - * @since 1.5.0 + * @group sort_funcs + * @since 2.1.0 */ - def expr(expr: String): Column = Column(internal.SqlExpression(expr)) + def asc_nulls_first(columnName: String): Column = Column(columnName).asc_nulls_first /** - * Call a SQL function. + * Returns a sort expression based on ascending order of the column, and null values appear + * after non-null values. + * {{{ + * df.sort(asc_nulls_last("dept"), desc("age")) + * }}} * - * @param funcName - * function name that follows the SQL identifier syntax (can be quoted, can be qualified) - * @param cols - * the expression parameters of function - * @group normal_funcs - * @since 3.5.0 + * @group sort_funcs + * @since 2.1.0 */ - @scala.annotation.varargs - def call_function(funcName: String, cols: Column*): Column = { - Column(internal.UnresolvedFunction(funcName, cols.map(_.node), isUserDefinedFunction = true)) - } - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Conditional Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def asc_nulls_last(columnName: String): Column = Column(columnName).asc_nulls_last /** - * Returns the first column that is not null, or null if all inputs are null. - * - * For example, `coalesce(a, b, c)` will return a if a is not null, or b if a is null and b is - * not null, or c if both a and b are null but c is not null. + * Returns a sort expression based on the descending order of the column. + * {{{ + * df.sort(asc("dept"), desc("age")) + * }}} * - * @param e - * the columns to work on. A column that evaluates to any type. - * @group conditional_funcs + * @group sort_funcs * @since 1.3.0 - * @return - * Returns a column of the same type as the input. */ - @scala.annotation.varargs - def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) + def desc(columnName: String): Column = Column(columnName).desc /** - * Returns col1 if it is not NaN, or col2 if col1 is NaN. - * - * Both inputs should be floating point columns (DoubleType or FloatType). + * Returns a sort expression based on the descending order of the column, and null values appear + * before non-null values. + * {{{ + * df.sort(asc("dept"), desc_nulls_first("age")) + * }}} * - * @param col1 - * the first column to check. A column that evaluates to a numeric. - * @param col2 - * the column to return if the first is NaN. A column that evaluates to a numeric. - * @group conditional_funcs - * @since 1.5.0 - * @return - * Returns a column of the same type as the first input. + * @group sort_funcs + * @since 2.1.0 */ - def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) + def desc_nulls_first(columnName: String): Column = Column(columnName).desc_nulls_first /** - * Evaluates a list of conditions and returns one of multiple possible result expressions. If - * otherwise is not defined at the end, null is returned for unmatched conditions. - * + * Returns a sort expression based on the descending order of the column, and null values appear + * after non-null values. * {{{ - * // Example: encoding gender string column into integer. - * - * // Scala: - * people.select(when(people("gender") === "male", 0) - * .when(people("gender") === "female", 1) - * .otherwise(2)) - * - * // Java: - * people.select(when(col("gender").equalTo("male"), 0) - * .when(col("gender").equalTo("female"), 1) - * .otherwise(2)) + * df.sort(asc("dept"), desc_nulls_last("age")) * }}} * - * @param condition - * the condition to evaluate. A column that evaluates to a boolean. - * @param value - * the value to return when the condition is true. A literal value, or a column expression. - * @group conditional_funcs - * @since 1.4.0 + * @group sort_funcs + * @since 2.1.0 + */ + def desc_nulls_last(columnName: String): Column = Column(columnName).desc_nulls_last + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Aggregate functions + ////////////////////////////////////////////////////////////////////////////////////////////// + + /** + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def when(condition: Column, value: Any): Column = - Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(e: Column): Column = approx_count_distinct(e) /** - * Returns `col2` if `col1` is null, or `col1` otherwise. - * - * @param col1 - * The column to test for null. A column of any type. - * @param col2 - * The column to return when col1 is null. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) /** - * Returns null if `col1` equals to `col2`, or `col1` otherwise. - * - * @param col1 - * The value to return if it is not equal to `col2`. A column of any type. - * @param col2 - * The value compared with `col1`. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) /** - * Returns null if `col` is equal to zero, or `col` otherwise. - * - * @param col - * The input value. A column that evaluates to a numeric. - * @group conditional_funcs - * @since 4.0.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def nullifzero(col: Column): Column = Column.fn("nullifzero", col) + @deprecated("Use approx_count_distinct", "2.1.0") + def approxCountDistinct(columnName: String, rsd: Double): Column = { + approx_count_distinct(Column(columnName), rsd) + } /** - * Returns `col2` if `col1` is null, or `col1` otherwise. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param col1 - * The value to return if it is not null. A column of any type. - * @param col2 - * The value to return if `col1` is null. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @param e + * The column to count distinct values in. A column of any type. + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) + def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) /** - * Returns `col2` if `col1` is not null, or `col3` otherwise. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param col1 - * The value that determines which branch to return. A column of any type. - * @param col2 - * The value to return if `col1` is not null. A column of any type. - * @param col3 - * The value to return if `col1` is null. A column of any type. - * @group conditional_funcs - * @since 3.5.0 + * @param columnName + * The name of the column to count distinct values in. A column of any type. + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) + def approx_count_distinct(columnName: String): Column = approx_count_distinct( + column(columnName)) /** - * Returns zero if `col` is null, or `col` otherwise. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param col - * The input value. A column that evaluates to a numeric. - * @group conditional_funcs - * @since 4.0.0 + * @param rsd + * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a + * double. Must be a constant. + * + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Predicate Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def approx_count_distinct(e: Column, rsd: Double): Column = { + Column.fn("approx_count_distinct", e, lit(rsd)) + } /** - * Return true iff the column is NaN. + * Aggregate function: returns the approximate number of distinct items in a group. * - * @param e - * the column to check. A column that evaluates to a numeric. - * @group predicate_funcs - * @since 1.6.0 + * @param rsd + * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a + * double. Must be a constant. + * + * @group agg_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a long. */ - def isnan(e: Column): Column = e.isNaN + def approx_count_distinct(columnName: String, rsd: Double): Column = { + approx_count_distinct(Column(columnName), rsd) + } /** - * Return true iff the column is null. + * Aggregate function: returns the average of the values in a group. * * @param e - * the column to check. A column that evaluates to any type. - * @group predicate_funcs - * @since 1.6.0 + * The column to average. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a numeric. */ - def isnull(e: Column): Column = e.isNull + def avg(e: Column): Column = Column.fn("avg", e) /** - * Inversion of boolean expression, i.e. NOT. - * {{{ - * // Scala: select rows that are not active (isActive === false) - * df.filter( !df("isActive") ) - * - * // Java: - * df.filter( not(df.col("isActive")) ); - * }}} + * Aggregate function: returns the average of the values in a group. * - * @param e - * the column to invert. A column that evaluates to a boolean. - * @group predicate_funcs + * @param columnName + * The name of the column to average. A column that evaluates to a numeric or interval. + * @group agg_funcs * @since 1.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a numeric. */ - def not(e: Column): Column = !e + def avg(columnName: String): Column = avg(Column(columnName)) /** - * Returns true if `str` matches `regexp`, or false otherwise. + * Aggregate function: returns a list of objects with duplicates. * - * @param str - * A column that evaluates to a string. - * @param regexp - * The regular expression pattern. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * @param e + * The column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) + def collect_list(e: Column): Column = Column.fn("collect_list", e) /** - * Returns true if `str` matches `regexp`, or false otherwise. + * Aggregate function: returns a list of objects with duplicates. * - * @param str - * A column that evaluates to a string. - * @param regexp - * The regular expression pattern. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * @param columnName + * The name of the column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) + def collect_list(columnName: String): Column = collect_list(Column(columnName)) /** - * Returns true if `str` matches `regexp`, or false otherwise. + * Aggregate function: returns a set of objects with duplicate elements eliminated. * - * @param str - * A column that evaluates to a string. - * @param regexp - * The regular expression pattern. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * @param e + * The column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) + def collect_set(e: Column): Column = Column.fn("collect_set", e) /** - * Returns true if str matches `pattern` with `escapeChar`, null if any arguments are null, - * false otherwise. + * Aggregate function: returns a set of objects with duplicate elements eliminated. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @param escapeChar - * The escape character. A column that evaluates to a string. Must be a constant. - * @group predicate_funcs - * @since 3.5.0 + * @param columnName + * The name of the column to collect. A column of any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def like(str: Column, pattern: Column, escapeChar: Column): Column = - Column.fn("like", str, pattern, escapeChar) + def collect_set(columnName: String): Column = collect_set(Column(columnName)) /** - * Returns true if str matches `pattern` with `escapeChar`('\'), null if any arguments are null, - * false otherwise. + * Aggregate function: returns the distinct union of the elements of an array-typed column + * across rows. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @group predicate_funcs - * @since 3.5.0 + * The aggregation buffer holds only the distinct elements, so its size is bounded by the + * element universe rather than by the number of input rows. Null elements are dropped by + * default (IGNORE NULLS), matching `collect_set`. With `RESPECT NULLS`, a single null element + * is kept, in which case this is equivalent to `array_distinct(flatten(collect_list(e)))`. The + * `RESPECT NULLS` clause is only available through SQL (e.g. + * `expr("collect_union(col) RESPECT NULLS")`). + * + * @param e + * The array column to collect the union of. A column of type array. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) + def collect_union(e: Column): Column = Column.fn("collect_union", e) /** - * Returns true if str matches `pattern` with `escapeChar` case-insensitively, null if any - * arguments are null, false otherwise. + * Aggregate function: returns the distinct union of the elements of an array-typed column + * across rows. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @param escapeChar - * The escape character. A column that evaluates to a string. Must be a constant. - * @group predicate_funcs - * @since 3.5.0 + * @param columnName + * The name of the array column to collect the union of. A column of type array. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to an array. */ - def ilike(str: Column, pattern: Column, escapeChar: Column): Column = - Column.fn("ilike", str, pattern, escapeChar) + def collect_union(columnName: String): Column = collect_union(Column(columnName)) /** - * Returns true if str matches `pattern` with `escapeChar`('\') case-insensitively, null if any - * arguments are null, false otherwise. + * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is + * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min + * sketch is a probabilistic data structure used for cardinality estimation using sub-linear + * space. * - * @param str - * A column that evaluates to a string. - * @param pattern - * The pattern to match. A column that evaluates to a string. - * @group predicate_funcs + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. + * @param seed + * The random seed. A column that evaluates to an integral. Must be a constant. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a binary. */ - def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) + def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = + Column.fn("count_min_sketch", e, eps, confidence, seed) /** - * Returns true if `col` is not null, or false otherwise. + * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is + * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min + * sketch is a probabilistic data structure used for cardinality estimation using sub-linear + * space. * - * @param col - * The column to check. A column of any type. - * @group predicate_funcs - * @since 3.5.0 + * @param e + * The column to compute the sketch on. A column that evaluates to an integral, string or + * binary. + * @param eps + * The relative error, must be positive. A column that evaluates to a numeric. Must be a + * constant. + * @param confidence + * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. + * Must be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a binary. */ - def isnotnull(col: Column): Column = Column.fn("isnotnull", col) + def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = + count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) /** - * Returns same result as the EQUAL(=) operator for non-null operands, but returns true if both - * are null, false if one of the them is null. + * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * - * @param col1 - * The first column to compare. A column of any type. - * @param col2 - * The second column to compare. A column of any type. - * @group predicate_funcs - * @since 3.5.0 + * @param column1 + * The first column. A column that evaluates to a numeric. + * @param column2 + * The second column. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a double. */ - def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Sort Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) /** - * Returns a sort expression based on ascending order of the column. - * {{{ - * df.sort(asc("dept"), desc("age")) - * }}} + * Aggregate function: returns the Pearson Correlation Coefficient for two columns. * - * @group sort_funcs - * @since 1.3.0 + * @param columnName1 + * The name of the first column. A column that evaluates to a numeric. + * @param columnName2 + * The name of the second column. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 + * @return + * Returns a column that evaluates to a double. */ - def asc(columnName: String): Column = Column(columnName).asc - - /** - * Returns a sort expression based on ascending order of the column, and null values return - * before non-null values. - * {{{ - * df.sort(asc_nulls_first("dept"), desc("age")) - * }}} - * - * @group sort_funcs - * @since 2.1.0 - */ - def asc_nulls_first(columnName: String): Column = Column(columnName).asc_nulls_first + def corr(columnName1: String, columnName2: String): Column = { + corr(Column(columnName1), Column(columnName2)) + } /** - * Returns a sort expression based on ascending order of the column, and null values appear - * after non-null values. - * {{{ - * df.sort(asc_nulls_last("dept"), desc("age")) - * }}} + * Aggregate function: returns the number of items in a group. * - * @group sort_funcs - * @since 2.1.0 + * @param e + * The column to count. A column of any type. + * @group agg_funcs + * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ - def asc_nulls_last(columnName: String): Column = Column(columnName).asc_nulls_last + def count(e: Column): Column = + Column.fn("count", e) /** - * Returns a sort expression based on the descending order of the column. - * {{{ - * df.sort(asc("dept"), desc("age")) - * }}} + * Aggregate function: returns the number of items in a group. * - * @group sort_funcs + * @param columnName + * The name of the column to count. A column of any type. + * @group agg_funcs * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ - def desc(columnName: String): Column = Column(columnName).desc + def count(columnName: String): TypedColumn[Any, Long] = + count(Column(columnName)).as(PrimitiveLongEncoder) /** - * Returns a sort expression based on the descending order of the column, and null values appear - * before non-null values. - * {{{ - * df.sort(asc("dept"), desc_nulls_first("age")) - * }}} + * Aggregate function: returns the number of distinct items in a group. * - * @group sort_funcs - * @since 2.1.0 - */ - def desc_nulls_first(columnName: String): Column = Column(columnName).desc_nulls_first - - /** - * Returns a sort expression based on the descending order of the column, and null values appear - * after non-null values. - * {{{ - * df.sort(asc("dept"), desc_nulls_last("age")) - * }}} + * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. * - * @group sort_funcs - * @since 2.1.0 + * @param expr + * The first column. A column of any type. + * @param exprs + * Additional columns. A column of any type. + * @group agg_funcs + * @since 1.3.0 + * @return + * Returns a column that evaluates to a long. */ - def desc_nulls_last(columnName: String): Column = Column(columnName).desc_nulls_last - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Mathematical Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + @scala.annotation.varargs + def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) /** - * Unary minus, i.e. negate the expression. - * {{{ - * // Select the amount column and negates all values. - * // Scala: - * df.select( -df("amount") ) + * Aggregate function: returns the number of distinct items in a group. * - * // Java: - * df.select( negate(df.col("amount")) ); - * }}} + * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. * - * @param e - * the column to negate. A column that evaluates to a numeric or interval. - * @group math_funcs + * @param columnName + * first column to compute on. A column of any type. + * @param columnNames + * additional columns to compute on. Columns of any type. + * @group agg_funcs * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def negate(e: Column): Column = -e + @scala.annotation.varargs + def countDistinct(columnName: String, columnNames: String*): Column = + count_distinct(Column(columnName), columnNames.map(Column.apply): _*) /** - * Generate a random column with independent and identically distributed (i.i.d.) samples - * uniformly distributed in [0.0, 1.0). - * - * @param seed - * the seed for the random generator. - * @note - * The function is non-deterministic in general case. + * Aggregate function: returns the number of distinct items in a group. * - * @group math_funcs - * @since 1.4.0 + * @param expr + * first column to compute on. A column of any type. + * @param exprs + * additional columns to compute on. Columns of any type. + * @group agg_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long. */ - def rand(seed: Long): Column = Column.fn("rand", lit(seed)) + @scala.annotation.varargs + def count_distinct(expr: Column, exprs: Column*): Column = + Column.fn("count", isDistinct = true, expr +: exprs: _*) /** - * Generate a random column with independent and identically distributed (i.i.d.) samples - * uniformly distributed in [0.0, 1.0). - * - * @note - * The function is non-deterministic in general case. + * Aggregate function: returns the population covariance for two columns. * - * @group math_funcs - * @since 1.4.0 + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return * Returns a column that evaluates to a double. */ - def rand(): Column = rand(SparkClassUtils.random.nextLong) + def covar_pop(column1: Column, column2: Column): Column = + Column.fn("covar_pop", column1, column2) /** - * Generate a column with independent and identically distributed (i.i.d.) samples from the - * standard normal distribution. - * - * @param seed - * the seed for the random generator. - * @note - * The function is non-deterministic in general case. + * Aggregate function: returns the population covariance for two columns. * - * @group math_funcs - * @since 1.4.0 + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return * Returns a column that evaluates to a double. */ - def randn(seed: Long): Column = Column.fn("randn", lit(seed)) + def covar_pop(columnName1: String, columnName2: String): Column = { + covar_pop(Column(columnName1), Column(columnName2)) + } /** - * Generate a column with independent and identically distributed (i.i.d.) samples from the - * standard normal distribution. - * - * @note - * The function is non-deterministic in general case. + * Aggregate function: returns the sample covariance for two columns. * - * @group math_funcs - * @since 1.4.0 + * @param column1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param column2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return * Returns a column that evaluates to a double. */ - def randn(): Column = randn(SparkClassUtils.random.nextLong) + def covar_samp(column1: Column, column2: Column): Column = + Column.fn("covar_samp", column1, column2) /** - * Computes the square root of the specified float value. + * Aggregate function: returns the sample covariance for two columns. * - * @param e - * the value to compute the square root of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.3.0 + * @param columnName1 + * first column to calculate covariance. A column that evaluates to a numeric. + * @param columnName2 + * second column to calculate covariance. A column that evaluates to a numeric. + * @group agg_funcs + * @since 2.0.0 * @return * Returns a column that evaluates to a double. */ - def sqrt(e: Column): Column = Column.fn("sqrt", e) + def covar_samp(columnName1: String, columnName2: String): Column = { + covar_samp(Column(columnName1), Column(columnName2)) + } /** - * Computes the square root of the specified float value. + * Aggregate function: returns the first value in a group. * - * @param colName - * the name of a numeric column to compute the square root of. - * @group math_funcs - * @since 1.5.0 + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param e + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def sqrt(colName: String): Column = sqrt(Column(colName)) + def first(e: Column, ignoreNulls: Boolean): Column = + Column.fn("first", false, e, lit(ignoreNulls)) /** - * Returns the sum of `left` and `right` and the result is null on overflow. The acceptable - * input types are the same with the `+` operator. + * Aggregate function: returns the first value of a column in a group. * - * @param left - * the left operand. A column that evaluates to a numeric or interval. - * @param right - * the right operand. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 3.5.0 + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param columnName + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 2.0.0 * @return * Returns a column of the same type as the input. */ - def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) + def first(columnName: String, ignoreNulls: Boolean): Column = { + first(Column(columnName), ignoreNulls) + } /** - * Returns `dividend``/``divisor`. It always performs floating point division. Its result is - * always null if `divisor` is 0. + * Aggregate function: returns the first value in a group. * - * @param left - * the dividend. A column that evaluates to a numeric or interval. - * @param right - * the divisor. A column that evaluates to a numeric. - * @group math_funcs - * @since 3.5.0 + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param e + * column to fetch the first value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return * Returns a column of the same type as the input. */ - def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) + def first(e: Column): Column = first(e, ignoreNulls = false) /** - * Returns the remainder of `dividend``/``divisor`. Its result is always null if `divisor` is 0. + * Aggregate function: returns the first value of a column in a group. * - * @param left - * the dividend. A column that evaluates to a numeric. - * @param right - * the divisor. A column that evaluates to a numeric. - * @group math_funcs - * @since 4.0.0 + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param columnName + * column to fetch the first value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return * Returns a column of the same type as the input. */ - def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) + def first(columnName: String): Column = first(Column(columnName)) /** - * Returns `left``*``right` and the result is null on overflow. The acceptable input types are - * the same with the `*` operator. + * Aggregate function: returns the first value in a group. * - * @param left - * the multiplicand. A column that evaluates to a numeric or interval. - * @param right - * the multiplier. A column that evaluates to a numeric or interval. - * @group math_funcs + * @param e + * column to fetch the first value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) + def first_value(e: Column): Column = Column.fn("first_value", e) /** - * Returns `left``-``right` and the result is null on overflow. The acceptable input types are - * the same with the `-` operator. + * Aggregate function: returns the first value in a group. * - * @param left - * the left operand. A column that evaluates to a numeric or interval. - * @param right - * the right operand. A column that evaluates to a numeric or interval. - * @group math_funcs + * The function by default returns the first values it sees. It will return the first non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param e + * column to fetch the first value for. A column of any type. + * @param ignoreNulls + * if first value is null then look for first non-null value. A column that evaluates to a + * boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) + def first_value(e: Column, ignoreNulls: Column): Column = + Column.fn("first_value", e, ignoreNulls) /** - * Computes the absolute value of a numeric value. + * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or + * not, returns 1 for aggregated or 0 for not aggregated in the result set. * * @param e - * the value to compute the absolute value of. A column that evaluates to a numeric or - * interval. - * @group math_funcs - * @since 1.3.0 + * column to check if it is aggregated. A column of any type. + * @group agg_funcs + * @since 2.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a byte. */ - def abs(e: Column): Column = Column.fn("abs", e) + def grouping(e: Column): Column = Column.fn("grouping", e) /** - * @param e - * the value to compute the inverse cosine of. A column that evaluates to a double. - * @return - * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column - * that evaluates to a double. + * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or + * not, returns 1 for aggregated or 0 for not aggregated in the result set. * - * @group math_funcs - * @since 1.4.0 + * @param columnName + * column to check if it is aggregated. A column of any type. + * @group agg_funcs + * @since 2.0.0 + * @return + * Returns a column that evaluates to a byte. */ - def acos(e: Column): Column = Column.fn("acos", e) + def grouping(columnName: String): Column = grouping(Column(columnName)) /** - * @param columnName - * the value to compute the inverse cosine of. + * Aggregate function: returns the level of grouping, equals to + * + * {{{ + * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) + * }}} + * + * @param cols + * columns to check for. Columns of any type. + * @note + * The list of columns should match with grouping columns exactly, or empty (means all the + * grouping columns). + * + * @group agg_funcs + * @since 2.0.0 * @return - * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column - * that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a long. */ - def acos(columnName: String): Column = acos(Column(columnName)) + @scala.annotation.varargs + def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) /** - * @param e - * the value to compute the inverse hyperbolic cosine of. A column that evaluates to a double. - * @return - * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. + * Aggregate function: returns the level of grouping, equals to * - * @group math_funcs - * @since 3.1.0 + * {{{ + * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) + * }}} + * + * @param colName + * the name of the first grouping column. A column of any type. + * @param colNames + * the names of the remaining grouping columns. Columns of any type. + * @note + * The list of columns should match with grouping columns exactly. + * + * @group agg_funcs + * @since 2.0.0 + * @return + * Returns a column that evaluates to a long. */ - def acosh(e: Column): Column = Column.fn("acosh", e) + @scala.annotation.varargs + def grouping_id(colName: String, colNames: String*): Column = { + grouping_id((Seq(colName) ++ colNames).map(n => Column(n)): _*) + } /** - * @param columnName - * the value to compute the inverse hyperbolic cosine of. + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with lgConfigK arg. + * + * @param e + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 3.1.0 + * Returns a column that evaluates to a binary. */ - def acosh(columnName: String): Column = acosh(Column(columnName)) + def hll_sketch_agg(e: Column, lgConfigK: Column): Column = + Column.fn("hll_sketch_agg", e, lgConfigK) /** + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with lgConfigK arg. + * * @param e - * the value to compute the inverse sine of. A column that evaluates to a double. + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column - * that evaluates to a double. - * - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def asin(e: Column): Column = Column.fn("asin", e) + def hll_sketch_agg(e: Column, lgConfigK: Int): Column = + Column.fn("hll_sketch_agg", e, lit(lgConfigK)) /** + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with lgConfigK arg. + * * @param columnName - * the value to compute the inverse sine of. + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. + * @param lgConfigK + * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column + * that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column - * that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def asin(columnName: String): Column = asin(Column(columnName)) + def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { + hll_sketch_agg(Column(columnName), lgConfigK) + } /** + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with default lgConfigK value. + * * @param e - * the value to compute the inverse hyperbolic sine of. A column that evaluates to a double. + * the column to compute the sketch on. A column that evaluates to an integral, a string or a + * binary. + * @group agg_funcs + * @since 3.5.0 * @return - * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. - * - * @group math_funcs - * @since 3.1.0 + * Returns a column that evaluates to a binary. */ - def asinh(e: Column): Column = Column.fn("asinh", e) + def hll_sketch_agg(e: Column): Column = + Column.fn("hll_sketch_agg", e) /** + * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch + * configured with default lgConfigK value. + * * @param columnName - * the value to compute the inverse hyperbolic sine of. + * the name of the column to compute the sketch on. A column that evaluates to an integral, a + * string or a binary. + * @group agg_funcs + * @since 3.5.0 * @return - * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 3.1.0 + * Returns a column that evaluates to a binary. */ - def asinh(columnName: String): Column = asinh(Column(columnName)) + def hll_sketch_agg(columnName: String): Column = { + hll_sketch_agg(Column(columnName)) + } /** + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values + * and allowDifferentLgConfigK is set to false. + * * @param e - * the value to compute the inverse tangent of. A column that evaluates to a double. - * @return - * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that - * evaluates to a double. - * - * @group math_funcs - * @since 1.4.0 - */ - def atan(e: Column): Column = Column.fn("atan", e) - - /** - * @param columnName - * the value to compute the inverse tangent of. + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column - * that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def atan(columnName: String): Column = atan(Column(columnName)) + def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = + Column.fn("hll_union_agg", e, allowDifferentLgConfigK) /** - * @param y - * coordinate on y-axis. A column that evaluates to a double. - * @param x - * coordinate on x-axis. A column that evaluates to a double. - * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values + * and allowDifferentLgConfigK is set to false. * - * @group math_funcs - * @since 1.4.0 - */ - def atan2(y: Column, x: Column): Column = Column.fn("atan2", y, x) - - /** - * @param y - * coordinate on y-axis - * @param xName - * coordinate on x-axis - * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 - */ - def atan2(y: Column, xName: String): Column = atan2(y, Column(xName)) - - /** - * @param yName - * coordinate on y-axis - * @param x - * coordinate on x-axis + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def atan2(yName: String, x: Column): Column = atan2(Column(yName), x) + def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = + Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) /** - * @param yName - * coordinate on y-axis - * @param xName - * coordinate on x-axis + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values + * and allowDifferentLgConfigK is set to false. + * + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. + * @param allowDifferentLgConfigK + * allow sketches with different lgConfigK values to be merged. A column that evaluates to a + * boolean. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def atan2(yName: String, xName: String): Column = - atan2(Column(yName), Column(xName)) + def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { + hll_union_agg(Column(columnName), allowDifferentLgConfigK) + } /** - * @param y - * coordinate on y-axis - * @param xValue - * coordinate on x-axis + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. + * + * @param e + * the column containing the HllSketch instances to merge. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 3.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def atan2(y: Column, xValue: Double): Column = atan2(y, lit(xValue)) + def hll_union_agg(e: Column): Column = + Column.fn("hll_union_agg", e) /** - * @param yName - * coordinate on y-axis - * @param xValue - * coordinate on x-axis + * Aggregate function: returns the updatable binary representation of the Datasketches + * HllSketch, generated by merging previously created Datasketches HllSketch instances via a + * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. + * + * @param columnName + * the name of the column containing the HllSketch instances to merge. A column that evaluates + * to a binary. + * @group agg_funcs + * @since 3.5.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def atan2(yName: String, xValue: Double): Column = atan2(Column(yName), xValue) + def hll_union_agg(columnName: String): Column = { + hll_union_agg(Column(columnName)) + } /** - * @param yValue - * coordinate on y-axis - * @param x - * coordinate on x-axis + * Aggregate function: returns the kurtosis of the values in a group. + * + * @param e + * the column to compute the kurtosis on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a double. */ - def atan2(yValue: Double, x: Column): Column = atan2(lit(yValue), x) + def kurtosis(e: Column): Column = Column.fn("kurtosis", e) /** - * @param yValue - * coordinate on y-axis - * @param xName - * coordinate on x-axis + * Aggregate function: returns the kurtosis of the values in a group. + * + * @param columnName + * the name of the column to compute the kurtosis on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * the theta component of the point (r, theta) in polar coordinates that - * corresponds to the point (x, y) in Cartesian coordinates, as if computed by - * `java.lang.Math.atan2`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a double. */ - def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName)) + def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) /** + * Aggregate function: returns the last value in a group. + * + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * * @param e - * target column to compute on. A column that evaluates to a numeric. - * @return - * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. + * the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. A + * column that evaluates to a boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. * - * @group math_funcs - * @since 3.1.0 + * @group agg_funcs + * @since 2.0.0 + * @return + * Returns a column of the same type as the input. */ - def atanh(e: Column): Column = Column.fn("atanh", e) + def last(e: Column, ignoreNulls: Boolean): Column = + Column.fn("last", false, e, lit(ignoreNulls)) /** + * Aggregate function: returns the last value of the column in a group. + * + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * * @param columnName - * target column to compute on. + * the name of the column to take the last value from. A column of any type. + * @param ignoreNulls + * if true, returns the last non-null value; if all values are null, null is returned. A + * column that evaluates to a boolean. Must be a constant. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 2.0.0 * @return - * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. - * @group math_funcs - * @since 3.1.0 + * Returns a column of the same type as the input. */ - def atanh(columnName: String): Column = atanh(Column(columnName)) + def last(columnName: String, ignoreNulls: Boolean): Column = { + last(Column(columnName), ignoreNulls) + } /** - * An expression that returns the string representation of the binary value of the given long - * column. For example, bin("12") returns "1100". + * Aggregate function: returns the last value in a group. + * + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. * * @param e - * target column to work on. A column that evaluates to an integral. - * @group math_funcs - * @since 1.5.0 + * column to fetch the last value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def bin(e: Column): Column = Column.fn("bin", e) + def last(e: Column): Column = last(e, ignoreNulls = false) /** - * An expression that returns the string representation of the binary value of the given long - * column. For example, bin("12") returns "1100". + * Aggregate function: returns the last value of the column in a group. * - * @param columnName - * target column to work on. - * @group math_funcs - * @since 1.5.0 + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def bin(columnName: String): Column = bin(Column(columnName)) + def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) /** - * Computes the cube-root of the given value. + * Aggregate function: returns the last value in a group. * * @param e - * target column to compute on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * column to fetch the last value for. A column of any type. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def cbrt(e: Column): Column = Column.fn("cbrt", e) + def last_value(e: Column): Column = Column.fn("last_value", e) /** - * Computes the cube-root of the given column. + * Aggregate function: returns the last value in a group. * - * @param columnName - * target column to compute on. - * @group math_funcs - * @since 1.4.0 + * The function by default returns the last values it sees. It will return the last non-null + * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * + * @param e + * column to fetch the last value for. A column of any type. + * @param ignoreNulls + * whether to skip null values. A column that evaluates to a boolean. + * @note + * The function is non-deterministic because its results depends on the order of the rows + * which may be non-deterministic after a shuffle. + * + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def cbrt(columnName: String): Column = cbrt(Column(columnName)) + def last_value(e: Column, ignoreNulls: Column): Column = + Column.fn("last_value", e, ignoreNulls) /** - * Computes the ceiling of the given value of `e` to `scale` decimal places. + * Create time from hour, minute and second fields. For invalid inputs it will throw an error. * - * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @param scale - * parameter to control the rounding behavior. A column that evaluates to an integral. Must be - * a constant. - * @group math_funcs - * @since 3.3.0 + * @param hour + * the hour to represent, from 0 to 23. A column that evaluates to an integer. + * @param minute + * the minute to represent, from 0 to 59. A column that evaluates to an integer. + * @param second + * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column that evaluates to a time. */ - def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) + def make_time(hour: Column, minute: Column, second: Column): Column = { + Column.fn("make_time", hour, minute, second) + } /** - * Computes the ceiling of the given value of `e` to 0 decimal places. + * Aggregate function: returns the most frequent value in a group. * * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * target column to compute on. A column of any type. + * @group agg_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column of the same type as the input. */ - def ceil(e: Column): Column = Column.fn("ceil", e) + def mode(e: Column): Column = Column.fn("mode", e) /** - * Computes the ceiling of the given value of `columnName` to 0 decimal places. + * Aggregate function: returns the most frequent value in a group. * - * @param columnName - * the value to compute the ceiling on. - * @group math_funcs - * @since 1.4.0 + * When multiple values have the same greatest frequency then either any of values is returned + * if deterministic is false or is not defined, or the lowest value is returned if deterministic + * is true. + * + * @param e + * target column to compute on. A column of any type. + * @param deterministic + * if there are multiple equally-frequent results then return the lowest. A boolean. Must be a + * constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column of the same type as the input. */ - def ceil(columnName: String): Column = ceil(Column(columnName)) + def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) /** - * Computes the ceiling of the given value of `e` to `scale` decimal places. + * Aggregate function: returns the maximum value of the expression in a group. * * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @param scale - * parameter to control the rounding behavior. A column that evaluates to an integer. Must be - * a constant. - * @group math_funcs - * @since 3.5.0 + * the target column on which the maximum value is computed. A column of any type. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column of the same type as the input. */ - def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) + def max(e: Column): Column = Column.fn("max", e) /** - * Computes the ceiling of the given value of `e` to 0 decimal places. + * Aggregate function: returns the maximum value of the column in a group. * - * @param e - * the value to compute the ceiling on. A column that evaluates to a numeric. - * @group math_funcs - * @since 3.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column of the same type as the input. */ - def ceiling(e: Column): Column = Column.fn("ceiling", e) + def max(columnName: String): Column = max(Column(columnName)) /** - * Convert a number in a string column from one base to another. + * Aggregate function: returns the value associated with the maximum value of ord. * - * @param num - * a column to convert base for. A column that evaluates to a string. - * @param fromBase - * from base number. A column that evaluates to an integer. - * @param toBase - * to base number. A column that evaluates to an integer. - * @group math_funcs - * @since 1.5.0 + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @note + * The function is non-deterministic so the output order can be different for those associated + * the same values of `e`. + * + * @group agg_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def conv(num: Column, fromBase: Int, toBase: Int): Column = - Column.fn("conv", num, lit(fromBase), lit(toBase)) + def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) /** + * Aggregate function: returns an array of values associated with the top `k` values of `ord`. + * + * The result array contains values in descending order by their associated ordering values. + * Returns null if there are no non-null ordering values. + * * @param e - * angle in radians. A column that evaluates to a double. - * @return - * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that - * evaluates to a double. + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. An integer. Must be a constant. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. * - * @group math_funcs - * @since 1.4.0 + * @group agg_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to an array. */ - def cos(e: Column): Column = Column.fn("cos", e) + def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) /** - * @param columnName - * angle in radians. A column that evaluates to a double. + * Aggregate function: returns an array of values associated with the top `k` values of `ord`. + * + * The result array contains values in descending order by their associated ordering values. + * Returns null if there are no non-null ordering values. + * + * @param e + * the column representing the values to be returned. A column of any type. + * @param ord + * the column that needs to be maximized. A column of any orderable type. + * @param k + * the number of top values to return. A column that evaluates to an integer. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. + * + * @group agg_funcs + * @since 4.2.0 * @return - * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to an array. */ - def cos(columnName: String): Column = cos(Column(columnName)) + def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) /** - * @param e - * hyperbolic angle. A column that evaluates to a double. - * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column - * that evaluates to a double. + * Aggregate function: returns the average of the values in a group. Alias for avg. * - * @group math_funcs + * @param e + * target column to compute on. A column that evaluates to a numeric. + * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ - def cosh(e: Column): Column = Column.fn("cosh", e) + def mean(e: Column): Column = avg(e) /** - * @param columnName - * hyperbolic angle - * @return - * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column - * that evaluates to a double. - * @group math_funcs + * Aggregate function: returns the average of the values in a group. Alias for avg. + * + * @group agg_funcs * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ - def cosh(columnName: String): Column = cosh(Column(columnName)) + def mean(columnName: String): Column = avg(columnName) /** + * Aggregate function: returns the median of the values in a group. + * * @param e - * angle in radians. A column that evaluates to a double. - * @return - * cotangent of the angle. Returns a column that evaluates to a double. - * - * @group math_funcs - * @since 3.3.0 + * target column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.4.0 + * @return + * Returns a column that evaluates to a double. */ - def cot(e: Column): Column = Column.fn("cot", e) + def median(e: Column): Column = Column.fn("median", e) /** + * Aggregate function: returns the minimum value of the expression in a group. + * * @param e - * angle in radians. A column that evaluates to a double. + * the target column on which the minimum value is computed. A column of any type. + * @group agg_funcs + * @since 1.3.0 * @return - * cosecant of the angle. Returns a column that evaluates to a double. - * - * @group math_funcs - * @since 3.3.0 + * Returns a column of the same type as the input. */ - def csc(e: Column): Column = Column.fn("csc", e) + def min(e: Column): Column = Column.fn("min", e) /** - * Returns Euler's number. + * Aggregate function: returns the minimum value of the column in a group. * - * @group math_funcs - * @since 3.5.0 + * @param columnName + * the name of the column on which the minimum value is computed. A column of an orderable + * type. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def e(): Column = Column.fn("e") + def min(columnName: String): Column = min(Column(columnName)) /** - * Computes the exponential of the given value. + * Aggregate function: returns the value associated with the minimum value of ord. * * @param e - * target column to compute on. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @note + * The function is non-deterministic so the output order can be different for those associated + * the same values of `e`. + * + * @group agg_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def exp(e: Column): Column = Column.fn("exp", e) + def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) /** - * Computes the exponential of the given column. + * Aggregate function: returns an array of values associated with the bottom `k` values of + * `ord`. * - * @param columnName - * target column to compute on. - * @group math_funcs - * @since 1.4.0 + * The result array contains values in ascending order by their associated ordering values. + * Returns null if there are no non-null ordering values. + * + * @param e + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. An integer. Must be a constant. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. + * + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def exp(columnName: String): Column = exp(Column(columnName)) + def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) /** - * Computes the exponential of the given value minus one. + * Aggregate function: returns an array of values associated with the bottom `k` values of + * `ord`. + * + * The result array contains values in ascending order by their associated ordering values. + * Returns null if there are no non-null ordering values. * * @param e - * column to calculate exponential for. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * the column representing the values that will be returned. A column of any type. + * @param ord + * the column that needs to be minimized. A column of an orderable type. + * @param k + * the number of bottom values to return. A column that evaluates to an integral. Must be a + * constant. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle when there are ties in the + * ordering expression. + * @note + * The maximum value of `k` is 100000. + * + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def expm1(e: Column): Column = Column.fn("expm1", e) + def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) /** - * Computes the exponential of the given column minus one. + * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given + * percentage(s) with value range in [0.0, 1.0]. * - * @param columnName - * column name to calculate exponential for. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param e + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to a double. */ - def expm1(columnName: String): Column = expm1(Column(columnName)) + def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) /** - * Computes the factorial of the given value. + * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given + * percentage(s) with value range in [0.0, 1.0]. * * @param e - * a column to calculate factorial for. A column that evaluates to an integral. - * @group math_funcs - * @since 1.5.0 + * the column to compute the percentile on. A column that evaluates to a numeric or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param frequency + * the positive frequency with which to weight each value. A column that evaluates to an + * integral. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a double. */ - def factorial(e: Column): Column = Column.fn("factorial", e) + def percentile(e: Column, percentage: Column, frequency: Column): Column = + Column.fn("percentile", e, percentage, frequency) /** - * Computes the floor of the given value of `e` to `scale` decimal places. + * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is + * the smallest value in the ordered `col` values (sorted from least to greatest) such that no + * more than `percentage` of `col` values is less than the value or equal to that value. + * + * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating + * point value, it must be between 0.0 and 1.0. + * + * The accuracy parameter is a positive numeric literal which controls approximation accuracy at + * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the + * relative error of the approximation. * * @param e - * the target column to compute the floor on. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to control the rounding behavior. A column that evaluates to - * an integral. - * @group math_funcs - * @since 3.3.0 + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column of the same type as the input. */ - def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) + def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = + Column.fn("percentile_approx", e, percentage, accuracy) /** - * Computes the floor of the given value of `e` to 0 decimal places. + * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is + * the smallest value in the ordered `col` values (sorted from least to greatest) such that no + * more than `percentage` of `col` values is less than the value or equal to that value. + * + * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating + * point value, it must be between 0.0 and 1.0. + * + * The accuracy parameter is a positive numeric literal which controls approximation accuracy at + * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the + * relative error of the approximation. * * @param e - * the target column to compute the floor on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * the column to compute the approximate percentile on. A column that evaluates to a numeric + * or interval. + * @param percentage + * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an + * array. Must be a constant. + * @param accuracy + * a positive numeric literal that controls approximation accuracy at the cost of memory. A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column of the same type as the input. */ - def floor(e: Column): Column = Column.fn("floor", e) + def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { + Column.fn("approx_percentile", e, percentage, accuracy) + } /** - * Computes the floor of the given column value to 0 decimal places. + * Aggregate function: returns the product of all numerical elements in a group. * - * @param columnName - * the target column name to compute the floor on. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param e + * the column to compute the product on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a long or decimal. + * Returns a column that evaluates to a double. */ - def floor(columnName: String): Column = floor(Column(columnName)) + def product(e: Column): Column = Column.internalFn("product", e) /** - * Returns the greatest value of the list of values, skipping null values. This function takes - * at least 2 parameters. It will return null iff all parameters are null. + * Aggregate function: returns the skewness of the values in a group. * - * @param exprs - * columns to check for greatest value. A column that evaluates to any type. - * @group math_funcs - * @since 1.5.0 + * @param e + * the column to compute the skewness on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - @scala.annotation.varargs - def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) + def skewness(e: Column): Column = Column.fn("skewness", e) /** - * Returns the greatest value of the list of column names, skipping null values. This function - * takes at least 2 parameters. It will return null iff all parameters are null. + * Aggregate function: returns the skewness of the values in a group. * * @param columnName - * the first column name to check for greatest value. A column of a comparable type. - * @param columnNames - * the remaining column names to check for greatest value. Columns of a comparable type. - * @group math_funcs - * @since 1.5.0 + * the name of the column to compute the skewness on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - @scala.annotation.varargs - def greatest(columnName: String, columnNames: String*): Column = { - greatest((columnName +: columnNames).map(Column.apply): _*) - } - - /** - * Computes hex value of the given column. - * - * @param column - * target column to work on. A column that evaluates to an integral, string or binary. - * @group math_funcs - * @since 1.5.0 - * @return - * Returns a column that evaluates to a string. - */ - def hex(column: Column): Column = Column.fn("hex", column) + def skewness(columnName: String): Column = skewness(Column(columnName)) /** - * Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to - * the byte representation of number. + * Aggregate function: alias for `stddev_samp`. * - * @param column - * target column to work on. A column that evaluates to a string. - * @group math_funcs - * @since 1.5.0 + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def unhex(column: Column): Column = Column.fn("unhex", column) + def std(e: Column): Column = Column.fn("std", e) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: alias for `stddev_samp`. * - * @param l - * a leg. A column that evaluates to a numeric. - * @param r - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param e + * the column to compute the standard deviation on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a double. */ - def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) + def stddev(e: Column): Column = Column.fn("stddev", e) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: alias for `stddev_samp`. * - * @param l - * a leg. A column that evaluates to a numeric. - * @param rightName - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param columnName + * the name of the column to compute the standard deviation on. A column that evaluates to a + * numeric. + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a double. */ - def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) + def stddev(columnName: String): Column = stddev(Column(columnName)) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: returns the sample standard deviation of the expression in a group. * - * @param leftName - * a leg. A column that evaluates to a numeric. - * @param r - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param e + * the column to compute the sample standard deviation on. A column that evaluates to a + * numeric. + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a double. */ - def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) + def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: returns the sample standard deviation of the expression in a group. * - * @param leftName - * a leg. A column that evaluates to a numeric. - * @param rightName - * b leg. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param columnName + * Name of the column to compute the sample standard deviation on. A column that evaluates to + * a numeric. + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a double. */ - def hypot(leftName: String, rightName: String): Column = - hypot(Column(leftName), Column(rightName)) + def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: returns the population standard deviation of the expression in a group. * - * @param l - * a leg. A column that evaluates to a numeric. - * @param r - * b leg. A column that evaluates to a numeric. Must be a constant. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column to compute the population standard deviation on. A column that evaluates to a + * numeric. + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a double. */ - def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) + def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: returns the population standard deviation of the expression in a group. * - * @param leftName - * The a leg of the triangle. A column that evaluates to a numeric. - * @param r - * The b leg of the triangle. A column that evaluates to a numeric. Must be a constant. - * @group math_funcs - * @since 1.4.0 + * @param columnName + * Name of the column to compute the population standard deviation on. A column that evaluates + * to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a double. */ - def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) + def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: returns the sum of all values in the expression. * - * @param l - * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. - * @param r - * The b leg of the triangle. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column to sum. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a numeric or interval. */ - def hypot(l: Double, r: Column): Column = hypot(lit(l), r) + def sum(e: Column): Column = Column.fn("sum", e) /** - * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. + * Aggregate function: returns the sum of all values in the given column. * - * @param l - * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. - * @param rightName - * The b leg of the triangle. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param columnName + * Name of the column to sum. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a numeric or interval. */ - def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) + def sum(columnName: String): Column = sum(Column(columnName)) /** - * Returns the least value of the list of values, skipping null values. This function takes at - * least 2 parameters. It will return null iff all parameters are null. + * Aggregate function: returns the sum of distinct values in the expression. * - * @param exprs - * The values to be compared. Columns that evaluate to a comparable type. - * @group math_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a numeric or interval. */ - @scala.annotation.varargs - def least(exprs: Column*): Column = Column.fn("least", exprs: _*) + @deprecated("Use sum_distinct", "3.2.0") + def sumDistinct(e: Column): Column = sum_distinct(e) /** - * Returns the least value of the list of column names, skipping null values. This function - * takes at least 2 parameters. It will return null iff all parameters are null. + * Aggregate function: returns the sum of distinct values in the expression. * - * @param columnName - * The name of the first column to be compared. A column of a comparable type. - * @param columnNames - * The names of the remaining columns to be compared. Columns of a comparable type. - * @group math_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a numeric or interval. */ - @scala.annotation.varargs - def least(columnName: String, columnNames: String*): Column = { - least((columnName +: columnNames).map(Column.apply): _*) - } + @deprecated("Use sum_distinct", "3.2.0") + def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) /** - * Computes the natural logarithm of the given value. + * Aggregate function: returns the sum of distinct values in the expression. * * @param e - * The value to compute the natural logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 3.5.0 + * The column to sum distinct values of. A column that evaluates to a numeric or interval. + * @group agg_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a numeric or interval. */ - def ln(e: Column): Column = Column.fn("ln", e) + def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) /** - * Computes the natural logarithm of the given value. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input + * column via a Datasketches Intersection instance. * * @param e - * The value to compute the natural logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * The column of Datasketches ThetaSketch instances to intersect. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log(e: Column): Column = ln(e) + def theta_intersection_agg(e: Column): Column = + Column.fn("theta_intersection_agg", e) /** - * Computes the natural logarithm of the given column. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input + * volumn via a Datasketches Intersection instance. * * @param columnName - * The name of the column to compute the natural logarithm of. A column that evaluates to a - * numeric. - * @group math_funcs - * @since 1.4.0 + * Name of the column of Datasketches ThetaSketch instances to intersect. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log(columnName: String): Column = log(Column(columnName)) + def theta_intersection_agg(columnName: String): Column = + theta_intersection_agg(Column(columnName)) /** - * Returns the first argument-base logarithm of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the `lgNomEntries` nominal + * entries. * - * @param base - * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. - * @param a - * The value to compute the logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) + def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = + Column.fn("theta_sketch_agg", e, lgNomEntries) /** - * Returns the first argument-base logarithm of the second argument. - * - * @param base - * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. - * @param columnName - * The name of the column to compute the logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 - * @return - * Returns a column that evaluates to a double. - */ - def log(base: Double, columnName: String): Column = log(base, Column(columnName)) - - /** - * Computes the logarithm of the given value in base 10. + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the `lgNomEntries` nominal + * entries. * * @param e - * The value to compute the base-10 logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log10(e: Column): Column = Column.fn("log10", e) + def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = + Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) /** - * Computes the logarithm of the given value in base 10. + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the `lgNomEntries` nominal + * entries. * * @param columnName - * The name of the column to compute the base-10 logarithm of. A column that evaluates to a - * numeric. - * @group math_funcs - * @since 1.4.0 + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. + * @param lgNomEntries + * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and + * 26). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log10(columnName: String): Column = log10(Column(columnName)) + def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = + theta_sketch_agg(Column(columnName), lgNomEntries) /** - * Computes the natural logarithm of the given value plus one. + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the default value of 12 for + * `lgNomEntries`. * * @param e - * The value to compute the natural logarithm of the value plus one. A column that evaluates - * to a numeric. - * @group math_funcs - * @since 1.4.0 + * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, + * binary or array. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log1p(e: Column): Column = Column.fn("log1p", e) + def theta_sketch_agg(e: Column): Column = + Column.fn("theta_sketch_agg", e) /** - * Computes the natural logarithm of the given column plus one. + * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch + * built with the values in the input column and configured with the default value of 12 for + * `lgNomEntries`. * * @param columnName - * The name of the column to compute the natural logarithm of the value plus one. A column - * that evaluates to a numeric. - * @group math_funcs - * @since 1.4.0 + * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, + * string, binary or array. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log1p(columnName: String): Column = log1p(Column(columnName)) + def theta_sketch_agg(columnName: String): Column = + theta_sketch_agg(Column(columnName)) /** - * Computes the logarithm of the given column in base 2. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param expr - * The value to compute the base-2 logarithm of. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.5.0 + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log2(expr: Column): Column = Column.fn("log2", expr) + def theta_union_agg(e: Column, lgNomEntries: Column): Column = + Column.fn("theta_union_agg", e, lgNomEntries) /** - * Computes the logarithm of the given value in base 2. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. + * + * @param e + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. + */ + def theta_union_agg(e: Column, lgNomEntries: Int): Column = + Column.fn("theta_union_agg", e, lit(lgNomEntries)) + + /** + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * * @param columnName - * a column to calculate logarithm for. A column that evaluates to a double. - * @group math_funcs - * @since 1.5.0 + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def log2(columnName: String): Column = log2(Column(columnName)) + def theta_union_agg(columnName: String, lgNomEntries: Int): Column = + theta_union_agg(Column(columnName), lgNomEntries) /** - * Returns the negated value. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It is configured with the default value of 12 for + * `lgNomEntries`. * * @param e - * column to calculate negative value for. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 3.5.0 + * The column containing binary ThetaSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def negative(e: Column): Column = Column.fn("negative", e) + def theta_union_agg(e: Column): Column = + Column.fn("theta_union_agg", e) /** - * Returns Pi. + * Aggregate function: returns the compact binary representation of the Datasketches + * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column + * via a Datasketches Union instance. It is configured with the default value of 12 for + * `lgNomEntries`. * - * @group math_funcs - * @since 3.5.0 + * @param columnName + * The name of the column containing binary ThetaSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pi(): Column = Column.fn("pi") + def theta_union_agg(columnName: String): Column = + theta_union_agg(Column(columnName)) /** - * Returns the value. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. The mode parameter specifies + * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * * @param e - * input value column. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 3.5.0 + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def positive(e: Column): Column = Column.fn("positive", e) + def tuple_intersection_agg_double(e: Column, mode: Column): Column = + Column.fn("tuple_intersection_agg_double", e, mode) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. The mode parameter specifies + * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(l: Column, r: Column): Column = Column.fn("power", l, r) + def tuple_intersection_agg_double(e: Column, mode: String): Column = + Column.fn("tuple_intersection_agg_double", e, lit(mode)) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. The mode parameter specifies + * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). * - * @param l - * the base number. A column that evaluates to a double. - * @param rightName - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) + def tuple_intersection_agg_double(columnName: String, mode: String): Column = + tuple_intersection_agg_double(Column(columnName), mode) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. It is configured with the + * default mode of 'sum'. * - * @param leftName - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) + def tuple_intersection_agg_double(e: Column): Column = + Column.fn("tuple_intersection_agg_double", e) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by intersecting the Datasketches TupleSketch instances + * in the input column via a Datasketches Intersection instance. It is configured with the + * default mode of 'sum'. * - * @param leftName - * the base number. - * @param rightName - * the exponent number. - * @group math_funcs - * @since 1.4.0 + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) + def tuple_intersection_agg_double(columnName: String): Column = + tuple_intersection_agg_double(Column(columnName)) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(l: Column, r: Double): Column = pow(l, lit(r)) + def tuple_intersection_agg_integer(e: Column, mode: Column): Column = + Column.fn("tuple_intersection_agg_integer", e, mode) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param leftName - * the base number. - * @param r - * the exponent number. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) + def tuple_intersection_agg_integer(e: Column, mode: String): Column = + Column.fn("tuple_intersection_agg_integer", e, lit(mode)) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(l: Double, r: Column): Column = pow(lit(l), r) + def tuple_intersection_agg_integer(columnName: String, mode: String): Column = + tuple_intersection_agg_integer(Column(columnName), mode) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. It is configured with + * the default mode of 'sum'. * - * @param l - * the base number. - * @param rightName - * the exponent number. - * @group math_funcs - * @since 1.4.0 + * @param e + * The column containing binary TupleSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) + def tuple_intersection_agg_integer(e: Column): Column = + Column.fn("tuple_intersection_agg_integer", e) /** - * Returns the value of the first argument raised to the power of the second argument. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by intersecting the Datasketches TupleSketch + * instances in the input column via a Datasketches Intersection instance. It is configured with + * the default mode of 'sum'. * - * @param l - * the base number. A column that evaluates to a double. - * @param r - * the exponent number. A column that evaluates to a double. - * @group math_funcs - * @since 3.5.0 + * @param columnName + * The name of the column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def power(l: Column, r: Column): Column = Column.fn("power", l, r) + def tuple_intersection_agg_integer(columnName: String): Column = + tuple_intersection_agg_integer(Column(columnName)) /** - * Returns the positive value of dividend mod divisor. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param dividend - * the column that contains dividend, or the specified dividend value. A column that evaluates - * to a numeric. - * @param divisor - * the column that contains divisor, or the specified divisor value. A column that evaluates - * to a numeric. - * @group math_funcs - * @since 1.5.0 + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) + def tuple_sketch_agg_double( + key: Column, + summary: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_sketch_agg_double", key, summary, lgNomEntries, mode) /** - * Returns the double value that is closest in value to the argument and is equal to a - * mathematical integer. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param e - * target column to compute on. A column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def rint(e: Column): Column = Column.fn("rint", e) + def tuple_sketch_agg_double( + key: Column, + summary: Column, + lgNomEntries: Int, + mode: String): Column = + Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries), lit(mode)) /** - * Returns the double value that is closest in value to the argument and is equal to a - * mathematical integer. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * the numeric column name to round to the closest integer. - * @group math_funcs - * @since 1.4.0 + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def rint(columnName: String): Column = rint(Column(columnName)) + def tuple_sketch_agg_double( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int, + mode: String): Column = + tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) /** - * Returns the value of the column `e` rounded to 0 decimal places with HALF_UP round mode. - * - * @param e - * the value to round. A column that evaluates to a numeric. - * @group math_funcs - * @since 1.5.0 - * @return - * Returns a column of the same type as the input. - */ - def round(e: Column): Column = round(e, 0) - - /** - * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs - * @since 1.5.0 + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) + def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = + Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) /** - * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs - * @since 4.0.0 + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) + def tuple_sketch_agg_double( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int): Column = + tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) /** - * Truncates the value of `e` toward zero to 0 decimal places. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * the value to truncate. A column that evaluates to a numeric. + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to a + * numeric. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input, except that a decimal input may return a - * decimal of different precision and scale. - * @group math_funcs - * @since 4.4.0 + * Returns a column that evaluates to a binary. */ - def truncate(e: Column): Column = truncate(e, 0) + def tuple_sketch_agg_double(key: Column, summary: Column): Column = + Column.fn("tuple_sketch_agg_double", key, summary) /** - * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than - * or equal to 0, or to the left of the decimal point when `scale` is less than 0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * the value to truncate. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to keep. A column that evaluates to an integral. Must be a - * constant. + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input, except that a decimal input may return a - * decimal of different precision and scale. - * @group math_funcs - * @since 4.4.0 + * Returns a column that evaluates to a binary. */ - def truncate(e: Column, scale: Int): Column = Column.fn("truncate", e, lit(scale)) + def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = + tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) /** - * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than - * or equal to 0, or to the left of the decimal point when `scale` is less than 0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param e - * the value to truncate. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to keep. A column that evaluates to an integral. Must be a - * constant. + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input, except that a decimal input may return a - * decimal of different precision and scale. - * @group math_funcs - * @since 4.4.0 + * Returns a column that evaluates to a binary. */ - def truncate(e: Column, scale: Column): Column = Column.fn("truncate", e, scale) + def tuple_sketch_agg_integer( + key: Column, + summary: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_sketch_agg_integer", key, summary, lgNomEntries, mode) /** - * Returns the value of the column `e` rounded to 0 decimal places with HALF_EVEN round mode. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param e - * the value to round. A column that evaluates to a numeric. - * @group math_funcs - * @since 2.0.0 + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def bround(e: Column): Column = bround(e, 0) + def tuple_sketch_agg_integer( + key: Column, + summary: Column, + lgNomEntries: Int, + mode: String): Column = + Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries), lit(mode)) /** - * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter + * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs - * @since 2.0.0 + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) + def tuple_sketch_agg_integer( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int, + mode: String): Column = + tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) /** - * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is - * greater than or equal to 0 or at integral part when `scale` is less than 0. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @param e - * the value to round. A column that evaluates to a numeric. - * @param scale - * the number of decimal places to round to. A column that evaluates to an integral. Must be a - * constant. - * @group math_funcs - * @since 4.0.0 + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) + def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = + Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) /** - * @param e - * angle in radians. A column that evaluates to a double. - * @return - * secant of the angle. Returns a column that evaluates to a double. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns and + * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. * - * @group math_funcs - * @since 3.3.0 + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. + * @param lgNomEntries + * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a binary. */ - def sec(e: Column): Column = Column.fn("sec", e) + def tuple_sketch_agg_integer( + keyColumnName: String, + summaryColumnName: String, + lgNomEntries: Int): Column = + tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) /** - * Computes the signum of the given value. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * the value to compute the signum of. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 3.5.0 + * @param key + * the key values against which unique counting occurs. A column that evaluates to an array, a + * binary, a numeric, or a string. + * @param summary + * the summary values against which mode aggregations occur. A column that evaluates to an + * integral. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def sign(e: Column): Column = Column.fn("sign", e) + def tuple_sketch_agg_integer(key: Column, summary: Column): Column = + Column.fn("tuple_sketch_agg_integer", key, summary) /** - * Computes the signum of the given value. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary built with the key and summary values in the input columns. It + * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * the value to compute the signum of. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 1.4.0 + * @param keyColumnName + * the name of the column containing the key values against which unique counting occurs. A + * column that evaluates to an array, a binary, a numeric, or a string. + * @param summaryColumnName + * the name of the column containing the summary values against which mode aggregations occur. + * A column that evaluates to an integral. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def signum(e: Column): Column = Column.fn("signum", e) + def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = + tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) /** - * Computes the signum of the given column. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). * - * @param columnName - * column to compute the signum on. A column that evaluates to a numeric or interval. - * @group math_funcs - * @since 1.4.0 + * @param e + * the column containing binary TupleSketch representations to union. A column that evaluates + * to a binary. + * @param lgNomEntries + * the log-base-2 of nominal entries for the union buffer (must be between 4 and 26). A column + * that evaluates to an integral. Must be a constant. + * @param mode + * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def signum(columnName: String): Column = signum(Column(columnName)) + def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). + * * @param e - * angle in radians. A column that evaluates to a double. + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates - * to a double. - * - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def sin(e: Column): Column = Column.fn("sin", e) + def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). + * * @param columnName - * angle in radians. A column that evaluates to a double. + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates - * to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def sin(columnName: String): Column = sin(Column(columnName)) + def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = + tuple_union_agg_double(Column(columnName), lgNomEntries, mode) /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. + * * @param e - * hyperbolic angle. A column that evaluates to a double. + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def sinh(e: Column): Column = Column.fn("sinh", e) + def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. + * * @param columnName - * hyperbolic angle. A column that evaluates to a double. + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def sinh(columnName: String): Column = sinh(Column(columnName)) + def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = + tuple_union_agg_double(Column(columnName), lgNomEntries) /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. + * * @param e - * angle in radians. A column that evaluates to a double. + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @group agg_funcs + * @since 4.2.0 * @return - * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that - * evaluates to a double. - * - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def tan(e: Column): Column = Column.fn("tan", e) + def tuple_union_agg_double(e: Column): Column = + Column.fn("tuple_union_agg_double", e) /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with a double type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. + * * @param columnName - * angle in radians. A column that evaluates to a double. + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.2.0 * @return - * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def tan(columnName: String): Column = tan(Column(columnName)) + def tuple_union_agg_double(columnName: String): Column = + tuple_union_agg_double(Column(columnName)) /** + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). + * * @param e - * hyperbolic angle. A column that evaluates to a double. - * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 - */ - def tanh(e: Column): Column = Column.fn("tanh", e) - - /** - * @param columnName - * hyperbolic angle. A column that evaluates to a double. + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a - * column that evaluates to a double. - * @group math_funcs - * @since 1.4.0 + * Returns a column that evaluates to a binary. */ - def tanh(columnName: String): Column = tanh(Column(columnName)) + def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) /** - * @group math_funcs - * @since 1.4.0 + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). + * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - @deprecated("Use degrees", "2.1.0") - def toDegrees(e: Column): Column = degrees(e) + def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) /** - * @group math_funcs - * @since 1.4.0 + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric + * summaries (sum, min, max, alwaysone). + * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @param mode + * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a + * string. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - @deprecated("Use degrees", "2.1.0") - def toDegrees(columnName: String): Column = degrees(Column(columnName)) + def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = + tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) /** - * Converts an angle measured in radians to an approximately equivalent angle measured in - * degrees. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * * @param e - * angle in radians. A column that evaluates to a double. + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that - * evaluates to a double. - * - * @group math_funcs - * @since 2.1.0 + * Returns a column that evaluates to a binary. */ - def degrees(e: Column): Column = Column.fn("degrees", e) + def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) /** - * Converts an angle measured in radians to an approximately equivalent angle measured in - * degrees. + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It allows the configuration of + * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. * * @param columnName - * angle in radians. A column that evaluates to a double. + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group agg_funcs + * @since 4.2.0 * @return - * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 2.1.0 + * Returns a column that evaluates to a binary. */ - def degrees(columnName: String): Column = degrees(Column(columnName)) + def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = + tuple_union_agg_integer(Column(columnName), lgNomEntries) /** - * @group math_funcs - * @since 1.4.0 + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. + * + * @param e + * The input column containing binary TupleSketch representations. A column that evaluates to + * a binary. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - @deprecated("Use radians", "2.1.0") - def toRadians(e: Column): Column = radians(e) + def tuple_union_agg_integer(e: Column): Column = + Column.fn("tuple_union_agg_integer", e) /** - * @group math_funcs - * @since 1.4.0 + * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch + * with an integer type summary, generated by the union of Datasketches TupleSketch instances in + * the input column via a Datasketches Union instance. It is configured with the default values + * of 12 for `lgNomEntries` and 'sum' for mode. + * + * @param columnName + * The name of the input column containing binary TupleSketch representations. A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - @deprecated("Use radians", "2.1.0") - def toRadians(columnName: String): Column = radians(Column(columnName)) + def tuple_union_agg_integer(columnName: String): Column = + tuple_union_agg_integer(Column(columnName)) /** - * Converts an angle measured in degrees to an approximately equivalent angle measured in - * radians. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * * @param e - * angle in degrees. A column that evaluates to a double. + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that - * evaluates to a double. - * - * @group math_funcs - * @since 2.1.0 + * Returns a column that evaluates to a binary. */ - def radians(e: Column): Column = Column.fn("radians", e) + def kll_sketch_agg_bigint(e: Column, k: Column): Column = + Column.fn("kll_sketch_agg_bigint", e, k) /** - * Converts an angle measured in degrees to an approximately equivalent angle measured in - * radians. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param columnName - * angle in degrees. A column that evaluates to a double. + * @param e + * The input column containing the values to aggregate. A column that evaluates to an + * integral. + * @param k + * The parameter that controls the size and accuracy of the sketch. A column that evaluates to + * an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that - * evaluates to a double. - * @group math_funcs - * @since 2.1.0 + * Returns a column that evaluates to a binary. */ - def radians(columnName: String): Column = radians(Column(columnName)) + def kll_sketch_agg_bigint(e: Column, k: Int): Column = + Column.fn("kll_sketch_agg_bigint", e, lit(k)) /** - * Returns the bucket number into which the value of this expression would fall after being - * evaluated. Note that input arguments must follow conditions listed below; otherwise, the - * method will return null. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param v - * value to compute a bucket number in the histogram. A column that evaluates to a double or - * interval. - * @param min - * minimum value of the histogram. A column that evaluates to a double or interval. - * @param max - * maximum value of the histogram. A column that evaluates to a double or interval. - * @param numBucket - * the number of buckets. A column that evaluates to a long. + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * the bucket number into which the value would fall after being evaluated. Returns a column - * that evaluates to a long. - * @group math_funcs - * @since 3.5.0 + * Returns a column that evaluates to a binary. */ - def width_bucket(v: Column, min: Column, max: Column, numBucket: Column): Column = - Column.fn("width_bucket", v, min, max, numBucket) + def kll_sketch_agg_bigint(columnName: String, k: Int): Column = + kll_sketch_agg_bigint(Column(columnName), k) /** - * Returns a random value with independent and identically distributed (i.i.d.) values with the - * specified range of numbers. The provided numbers specifying the minimum and maximum values of - * the range must be constant. If both of these numbers are integers, then the result will also - * be an integer. Otherwise if one or both of these are floating-point numbers, then the result - * will also be a floating-point number. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column with default k value of 200. * - * @param min - * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. - * @param max - * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. - * @group math_funcs - * @since 4.0.0 + * @param e + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def uniform(min: Column, max: Column): Column = - uniform(min, max, lit(SparkClassUtils.random.nextLong)) + def kll_sketch_agg_bigint(e: Column): Column = + Column.fn("kll_sketch_agg_bigint", e) /** - * Returns a random value with independent and identically distributed (i.i.d.) values with the - * specified range of numbers, with the chosen random seed. The provided numbers specifying the - * minimum and maximum values of the range must be constant. If both of these numbers are - * integers, then the result will also be an integer. Otherwise if one or both of these are - * floating-point numbers, then the result will also be a floating-point number. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllLongsSketch built with the values in the input column with default k value of 200. * - * @param min - * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. - * @param max - * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. - * @param seed - * Random number seed to use. A column that evaluates to an integral. Must be a constant. - * @group math_funcs - * @since 4.0.0 + * @param columnName + * The column containing bigint values to aggregate. A column that evaluates to an integral. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def uniform(min: Column, max: Column, seed: Column): Column = - Column.fn("uniform", min, max, seed) + def kll_sketch_agg_bigint(columnName: String): Column = + kll_sketch_agg_bigint(Column(columnName)) /** - * Returns a random value with independent and identically distributed (i.i.d.) uniformly - * distributed values in [0, 1). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param seed - * Random number seed to use. A column that evaluates to an integral. Must be a constant. - * @group math_funcs - * @since 3.5.0 + * @param e + * The column containing float values to aggregate. A column that evaluates to a float. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def random(seed: Column): Column = Column.fn("random", seed) + def kll_sketch_agg_float(e: Column, k: Column): Column = + Column.fn("kll_sketch_agg_float", e, k) /** - * Returns a random value with independent and identically distributed (i.i.d.) uniformly - * distributed values in [0, 1). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @group math_funcs - * @since 3.5.0 + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def random(): Column = random(lit(SparkClassUtils.random.nextLong)) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // String Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def kll_sketch_agg_float(e: Column, k: Int): Column = + Column.fn("kll_sketch_agg_float", e, lit(k)) /** - * Returns a string of the specified length whose characters are chosen uniformly at random from - * the following pool of characters: 0-9, a-z, A-Z. The string length must be a constant - * two-byte or four-byte integer (SMALLINT or INT, respectively). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param length - * the number of characters in the string to generate. A column that evaluates to an integral. - * Must be a constant. - * @group string_funcs - * @since 4.0.0 + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def randstr(length: Column): Column = - randstr(length, lit(SparkClassUtils.random.nextLong)) + def kll_sketch_agg_float(columnName: String, k: Int): Column = + kll_sketch_agg_float(Column(columnName), k) /** - * Returns a string of the specified length whose characters are chosen uniformly at random from - * the following pool of characters: 0-9, a-z, A-Z, with the chosen random seed. The string - * length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column with default k value of 200. * - * @param length - * the number of characters in the string to generate. A column that evaluates to an integral. - * Must be a constant. - * @param seed - * the random seed to use. A column that evaluates to an integral. - * @group string_funcs - * @since 4.0.0 + * @param e + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) + def kll_sketch_agg_float(e: Column): Column = + Column.fn("kll_sketch_agg_float", e) /** - * Computes the numeric value of the first character of the string column, and returns the - * result as an int column. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllFloatsSketch built with the values in the input column with default k value of 200. * - * @param e - * The target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * @param columnName + * The column containing float values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def ascii(e: Column): Column = Column.fn("ascii", e) + def kll_sketch_agg_float(columnName: String): Column = + kll_sketch_agg_float(Column(columnName)) /** - * Computes the BASE64 encoding of a binary column and returns it as a string column. This is - * the reverse of unbase64. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * * @param e - * The target column to work on. A column that evaluates to a binary. - * @group string_funcs - * @since 1.5.0 + * The column containing double values to aggregate. A column that evaluates to a float or + * double. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def base64(e: Column): Column = Column.fn("base64", e) + def kll_sketch_agg_double(e: Column, k: Column): Column = + Column.fn("kll_sketch_agg_double", e, k) /** - * Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a string column. - * This is the reverse of from_base32. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * * @param e - * The target column to work on. A column that evaluates to a binary. - * @group string_funcs - * @since 4.3.0 + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def to_base32(e: Column): Column = Column.fn("to_base32", e) + def kll_sketch_agg_double(e: Column, k: Int): Column = + Column.fn("kll_sketch_agg_double", e, lit(k)) /** - * Calculates the bit length for the specified string column. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column. The optional k parameter controls + * the size and accuracy of the sketch (default 200, range 8-65535). * - * @param e - * The source column or strings. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.3.0 + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @param k + * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that + * evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def bit_length(e: Column): Column = Column.fn("bit_length", e) + def kll_sketch_agg_double(columnName: String, k: Int): Column = + kll_sketch_agg_double(Column(columnName), k) /** - * Concatenates multiple input string columns together into a single string column, using the - * given separator. - * - * @param sep - * The words separator. A column that evaluates to a string. Must be a constant. - * @param exprs - * The list of columns to work on. Each a column that evaluates to a string or an array of - * strings. - * @note - * Input strings which are null are skipped. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column with default k value of 200. * - * @group string_funcs - * @since 1.5.0 + * @param e + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - @scala.annotation.varargs - def concat_ws(sep: String, exprs: Column*): Column = - Column.fn("concat_ws", lit(sep) +: exprs: _*) + def kll_sketch_agg_double(e: Column): Column = + Column.fn("kll_sketch_agg_double", e) /** - * Computes the first argument into a string from a binary using the provided character set (one - * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either - * argument is null, the result will also be null. + * Aggregate function: returns the compact binary representation of the Datasketches + * KllDoublesSketch built with the values in the input column with default k value of 200. * - * @param value - * The target column to work on. A column that evaluates to a binary. - * @param charset - * The charset to use to decode to. A column that evaluates to a string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param columnName + * The column containing double values to aggregate. A column that evaluates to a numeric. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def decode(value: Column, charset: String): Column = - Column.fn("decode", value, lit(charset)) + def kll_sketch_agg_double(columnName: String): Column = + kll_sketch_agg_double(Column(columnName)) /** - * Computes the first argument into a binary from a string using the provided character set (one - * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either - * argument is null, the result will also be null. + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range + * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param value - * The target column to work on. A column that evaluates to a string. - * @param charset - * The charset to use to encode. A column that evaluates to a string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return * Returns a column that evaluates to a binary. */ - def encode(value: Column, charset: String): Column = - Column.fn("encode", value, lit(charset)) + def kll_merge_agg_bigint(e: Column, k: Column): Column = + Column.fn("kll_merge_agg_bigint", e, k) /** - * Returns true if the input is a valid UTF-8 string, otherwise returns false. + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range + * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param str - * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a - * string. - * @group string_funcs - * @since 4.0.0 + * @param e + * The column containing binary KllLongsSketch representations to merge. A column that + * evaluates to a binary. + * @param k + * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A + * column that evaluates to an integral. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a binary. */ - def is_valid_utf8(str: Column): Column = - Column.fn("is_valid_utf8", str) - + def kll_merge_agg_bigint(e: Column, k: Int): Column = + Column.fn("kll_merge_agg_bigint", e, lit(k)) + /** - * Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the - * Unicode replacement character (U+FFFD). + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range + * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param str - * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a - * string. - * @group string_funcs - * @since 4.0.0 + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def make_valid_utf8(str: Column): Column = - Column.fn("make_valid_utf8", str) + def kll_merge_agg_bigint(columnName: String, k: Int): Column = + kll_merge_agg_bigint(Column(columnName), k) /** - * Returns the input value if it corresponds to a valid UTF-8 string, or emits a - * SparkIllegalArgumentException exception otherwise. + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param str - * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a - * string. - * @group string_funcs - * @since 4.0.0 + * @param e + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def validate_utf8(str: Column): Column = - Column.fn("validate_utf8", str) + def kll_merge_agg_bigint(e: Column): Column = + Column.fn("kll_merge_agg_bigint", e) /** - * Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. + * Aggregate function: merges binary KllLongsSketch representations and returns the merged + * sketch. If k is not specified, the merged sketch adopts the k value from the first input + * sketch. * - * @param str - * the input value. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * @param columnName + * The column containing binary KllLongsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def try_validate_utf8(str: Column): Column = - Column.fn("try_validate_utf8", str) + def kll_merge_agg_bigint(columnName: String): Column = + kll_merge_agg_bigint(Column(columnName)) /** - * Returns the Unicode normalization of `str` using the given normalization `form`. Valid forms - * are 'NFC', 'NFD', 'NFKC', and 'NFKD', as defined by Unicode Standard Annex #15. The form name - * is case-insensitive. Normalization is backed by Spark's bundled ICU4J library rather than the - * JVM's own Unicode data, so results are stable across JVM vendors and versions. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param str - * the input string to normalize. - * @param form - * the normalization form: 'NFC', 'NFD', 'NFKC', or 'NFKD'. - * @group string_funcs - * @since 4.4.0 + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integral. + * @group agg_funcs + * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ - def normalize(str: Column, form: Column): Column = - Column.fn("normalize", str, form) + def kll_merge_agg_float(e: Column, k: Column): Column = + Column.fn("kll_merge_agg_float", e, k) /** - * Returns the Unicode normalization of `str` using the default form 'NFC'. To use a different - * form, call the two-argument overload. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param str - * the input string to normalize. - * @group string_funcs - * @since 4.4.0 + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 + * @return + * Returns a column that evaluates to a binary. */ - def normalize(str: Column): Column = - Column.fn("normalize", str) + def kll_merge_agg_float(e: Column, k: Int): Column = + Column.fn("kll_merge_agg_float", e, lit(k)) /** - * Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places with - * HALF_EVEN round mode, and returns the result as a string column. - * - * If d is 0, the result has no decimal point or fractional part. If d is less than 0, the - * result will be null. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param x - * the numeric value to be formatted. A column that evaluates to a numeric. - * @param d - * the number of decimal places. A column that evaluates to an integral. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) + def kll_merge_agg_float(columnName: String, k: Int): Column = + kll_merge_agg_float(Column(columnName), k) /** - * Formats the arguments in printf-style and returns the result as a string column. + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param format - * the format string that can contain embedded format tags. A column that evaluates to a - * string. Must be a constant. - * @param arguments - * the values to be used in formatting. Columns that evaluate to any type. - * @group string_funcs - * @since 1.5.0 + * @param e + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - @scala.annotation.varargs - def format_string(format: String, arguments: Column*): Column = - Column.fn("format_string", lit(format) +: arguments: _*) + def kll_merge_agg_float(e: Column): Column = + Column.fn("kll_merge_agg_float", e) /** - * Returns a new string column by converting the first letter of each word to uppercase. Words - * are delimited by whitespace. - * - * For example, "hello world" will become "Hello World". + * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param e - * the target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * @param columnName + * The column containing binary KllFloatsSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def initcap(e: Column): Column = Column.fn("initcap", e) + def kll_merge_agg_float(columnName: String): Column = + kll_merge_agg_float(Column(columnName)) /** - * Locate the position of the first occurrence of substr column in the given string. Returns - * null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. Must be a constant. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @group string_funcs - * @since 1.5.0 + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def instr(str: Column, substring: String): Column = instr(str, lit(substring)) + def kll_merge_agg_double(e: Column, k: Column): Column = + Column.fn("kll_merge_agg_double", e, k) /** - * Locate the position of the first occurrence of substr column in the given string. Returns - * null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @group string_funcs - * @since 4.0.0 + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) + def kll_merge_agg_double(e: Column, k: Int): Column = + Column.fn("kll_merge_agg_double", e, lit(k)) /** - * Locate the position of the first occurrence of `substring` in `str`, starting the search from - * position `start`. Returns null if either of the arguments are null. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. Must be a - * constant. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. - * - * @group string_funcs - * @since 4.3.0 + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @param k + * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to + * an integer. Must be a constant. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def instr(str: Column, substring: Column, start: Int): Column = - Column.fn("instr", str, substring, lit(start)) + def kll_merge_agg_double(columnName: String, k: Int): Column = + kll_merge_agg_double(Column(columnName), k) /** - * Locate the position of the first occurrence of `substring` in `str`, starting the search from - * position `start`. Returns null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @group string_funcs - * @since 4.3.0 + * @param e + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def instr(str: Column, substring: Column, start: Column): Column = - Column.fn("instr", str, substring, start) + def kll_merge_agg_double(e: Column): Column = + Column.fn("kll_merge_agg_double", e) /** - * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the - * search from position `start`. Returns null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. Must be a - * constant. - * @param occurrence - * which occurrence of the substring to locate. A column that evaluates to an integral. Must - * be a constant. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. - * @note - * The `occurrence` parameter must be a positive integer. + * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. + * If k is not specified, the merged sketch adopts the k value from the first input sketch. * - * @group string_funcs - * @since 4.3.0 + * @param columnName + * The column containing binary KllDoublesSketch representations. A column that evaluates to a + * binary. + * @group agg_funcs + * @since 4.1.2 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = - Column.fn("instr", str, substring, lit(start), lit(occurrence)) + def kll_merge_agg_double(columnName: String): Column = + kll_merge_agg_double(Column(columnName)) /** - * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the - * search from position `start`. Returns null if either of the arguments are null. - * - * @param str - * the string to search in. A column that evaluates to a string. - * @param substring - * the substring to search for. A column that evaluates to a string. - * @param start - * the position to start the search from. A column that evaluates to an integral. - * @param occurrence - * which occurrence of the substring to locate. A column that evaluates to an integral. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. - * @note - * If `start` is positive, the search proceeds forward. If `start` is negative, the search - * proceeds backward from the end of the string. If `start` is 0, returns 0. - * @note - * The `occurrence` parameter must be a positive integer. + * Aggregate function: returns the concatenation of non-null input values. * - * @group string_funcs - * @since 4.3.0 + * @param e + * The target column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = - Column.fn("instr", str, substring, start, occurrence) + def listagg(e: Column): Column = Column.fn("listagg", e) /** - * Computes the character length of a given string or number of bytes of a binary string. The - * length of character strings include the trailing spaces. The length of binary strings - * includes binary zeros. + * Aggregate function: returns the concatenation of non-null input values, separated by the + * delimiter. * * @param e - * the target column to work on. A column that evaluates to a string or binary. - * @group string_funcs - * @since 1.5.0 + * The target column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * The delimiter used to separate the values. A column that evaluates to a string or binary. + * Must be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def length(e: Column): Column = Column.fn("length", e) + def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) /** - * Computes the character length of a given string or number of bytes of a binary string. The - * length of character strings include the trailing spaces. The length of binary strings - * includes binary zeros. + * Aggregate function: returns the concatenation of distinct non-null input values. * * @param e - * the target column to work on. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * the column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def len(e: Column): Column = Column.fn("len", e) + def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) /** - * Converts a string column to lower case. + * Aggregate function: returns the concatenation of distinct non-null input values, separated by + * the delimiter. * * @param e - * the target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.3.0 + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. + * @group agg_funcs + * @since 4.0.0 * @return * Returns a column that evaluates to a string. */ - def lower(e: Column): Column = Column.fn("lower", e) + def listagg_distinct(e: Column, delimiter: Column): Column = + Column.fn("listagg", isDistinct = true, e, delimiter) /** - * Computes the Levenshtein distance of the two given string columns if it's less than or equal - * to a given threshold. - * @param l - * the first input column. A column that evaluates to a string. - * @param r - * the second input column. A column that evaluates to a string. - * @param threshold - * the maximum distance to compute. A column that evaluates to an integral. Must be a - * constant. + * Aggregate function: returns the concatenation of non-null input values. Alias for `listagg`. + * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * result distance, or -1. Returns a column that evaluates to an integer. - * @group string_funcs - * @since 3.5.0 + * Returns a column of the same type as the input. */ - def levenshtein(l: Column, r: Column, threshold: Int): Column = - Column.fn("levenshtein", l, r, lit(threshold)) + def string_agg(e: Column): Column = Column.fn("string_agg", e) /** - * Computes the Levenshtein distance of the two given string columns. - * @param l - * the first input column. A column that evaluates to a string. - * @param r - * the second input column. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * Aggregate function: returns the concatenation of non-null input values, separated by the + * delimiter. Alias for `listagg`. + * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) + def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) /** - * Computes the Jaro-Winkler similarity between the two given string columns. The result is a - * double between 0.0 (no similarity) and 1.0 (identical). - * @param l - * A column that evaluates to a string. - * @param r - * A column that evaluates to a string. - * @group string_funcs - * @since 4.3.0 + * Aggregate function: returns the concatenation of distinct non-null input values. Alias for + * `listagg`. + * + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def jaro_winkler_similarity(l: Column, r: Column): Column = - Column.fn("jaro_winkler_similarity", l, r) + def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) /** - * Locate the position of the first occurrence of substr. - * - * @param substr - * The substring to find. A column that evaluates to a string. - * @param str - * A column that evaluates to a string. - * @note - * The position is not zero based, but 1 based index. Returns 0 if substr could not be found - * in str. + * Aggregate function: returns the concatenation of distinct non-null input values, separated by + * the delimiter. Alias for `listagg`. * - * @group string_funcs - * @since 1.5.0 + * @param e + * the column to compute on. A column that evaluates to a string or binary. + * @param delimiter + * the delimiter to separate the values. A column that evaluates to a string or binary. Must + * be a constant. + * @group agg_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a string. */ - def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) + def string_agg_distinct(e: Column, delimiter: Column): Column = + Column.fn("string_agg", isDistinct = true, e, delimiter) /** - * Locate the position of the first occurrence of substr in a string column, after position pos. + * Aggregate function: alias for `var_samp`. * - * @param substr - * The substring to find. A column that evaluates to a string. - * @param str - * A column that evaluates to a string. - * @param pos - * The starting position. A column that evaluates to an integer. - * @note - * The position is not zero based, but 1 based index. returns 0 if substr could not be found - * in str. - * - * @group string_funcs - * @since 1.5.0 + * @param e + * the column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def locate(substr: String, str: Column, pos: Int): Column = - Column.fn("locate", lit(substr), str, lit(pos)) + def variance(e: Column): Column = Column.fn("variance", e) /** - * Left-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. + * Aggregate function: alias for `var_samp`. * - * @param str - * A column that evaluates to a string. - * @param len - * The length of the padded result. A column that evaluates to an integer. - * @param pad - * The padding string. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) + def variance(columnName: String): Column = variance(Column(columnName)) /** - * Left-pad the binary column with pad to a byte length of len. If the binary column is longer - * than len, the return value is shortened to len bytes. + * Aggregate function: returns the unbiased variance of the values in a group. * - * @param str - * A column that evaluates to a binary. - * @param len - * The byte length of the padded result. A column that evaluates to an integer. - * @param pad - * The padding bytes. A column that evaluates to a binary. - * @group string_funcs - * @since 3.3.0 + * @param e + * the column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) + def var_samp(e: Column): Column = Column.fn("var_samp", e) /** - * Left-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. + * Aggregate function: returns the unbiased variance of the values in a group. * - * @param str - * A column that evaluates to a string. - * @param len - * The length of the padded result. A column that evaluates to an integer. - * @param pad - * The padding string. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) + def var_samp(columnName: String): Column = var_samp(Column(columnName)) /** - * Trim the spaces from left end for the specified string value. + * Aggregate function: returns the population variance of the values in a group. * * @param e - * A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * the column to compute on. A column that evaluates to a numeric. + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def ltrim(e: Column): Column = Column.fn("ltrim", e) + def var_pop(e: Column): Column = Column.fn("var_pop", e) /** - * Trim the specified character string from left end for the specified string column. - * @param e - * A column that evaluates to a string. - * @param trimString - * The trim string. A column that evaluates to a string. - * @group string_funcs - * @since 2.3.0 + * Aggregate function: returns the population variance of the values in a group. + * + * @group agg_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) + def var_pop(columnName: String): Column = var_pop(Column(columnName)) /** - * Trim the specified character string from left end for the specified string column. - * @param e - * A column that evaluates to a string. - * @param trim - * The trim string. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * Aggregate function: returns the average of the independent variable for non-null pairs in a + * group, where `y` is the dependent variable and `x` is the independent variable. + * + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) + def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) /** - * Calculates the byte length for the specified string column. + * Aggregate function: returns the average of the dependent variable for non-null pairs in a + * group, where `y` is the dependent variable and `x` is the independent variable. * - * @param e - * A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.3.0 + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def octet_length(e: Column): Column = Column.fn("octet_length", e) + def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) /** - * Marks a given column with specified collation. + * Aggregate function: returns the number of non-null number pairs in a group, where `y` is the + * dependent variable and `x` is the independent variable. * - * @param e - * A column that evaluates to a string. - * @param collation - * The collation name. A column that evaluates to a string. Must be a constant. - * @group string_funcs - * @since 4.0.0 + * @param y + * the dependent variable. A column that evaluates to a numeric. + * @param x + * the independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) + def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) /** - * Returns the collation name of a given column. + * Aggregate function: returns the intercept of the univariate linear regression line for + * non-null pairs in a group, where `y` is the dependent variable and `x` is the independent + * variable. * - * @param e - * A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def collation(e: Column): Column = Column.fn("collation", e) + def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) /** - * Returns a count of the number of times that the regular expression pattern `regexp` is - * matched in the string `str`. + * Aggregate function: returns the coefficient of determination for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) + def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) /** - * Extract a specific group matched by a Java regex, from the specified string column. If the - * regex did not match, or the specified group did not match, an empty string is returned. if - * the specified group index exceeds the group count of regex, an IllegalArgumentException will - * be thrown. + * Aggregate function: returns the slope of the linear regression line for non-null pairs in a + * group, where `y` is the dependent variable and `x` is the independent variable. * - * @param e - * target column to work on. A column that evaluates to a string. - * @param exp - * regex pattern to apply. A string. Must be a constant. - * @param groupIdx - * matched group id. An integer. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = - Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) + def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) /** - * Extract all strings in the `str` that match the `regexp` expression and corresponding to the - * first regex group index. + * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a double. */ - def regexp_extract_all(str: Column, regexp: Column): Column = - Column.fn("regexp_extract_all", str, regexp) + def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) /** - * Extract all strings in the `str` that match the `regexp` expression and corresponding to the - * regex group index. + * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @param idx - * matched group id. A column that evaluates to an integer. - * @group string_funcs + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a double. */ - def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = - Column.fn("regexp_extract_all", str, regexp, idx) + def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) /** - * Replace all substrings of the specified string value that match regexp with rep. + * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, + * where `y` is the dependent variable and `x` is the independent variable. * - * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A string. Must be a constant. - * @param replacement - * replacement string. A string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param y + * The dependent variable. A column that evaluates to a numeric. + * @param x + * The independent variable. A column that evaluates to a numeric. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def regexp_replace(e: Column, pattern: String, replacement: String): Column = - regexp_replace(e, lit(pattern), lit(replacement)) + def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) /** - * Replace all substrings of the specified string value that match regexp with rep, starting at - * the specified position `pos`. + * Aggregate function: returns some value of `e` for a group of rows. * * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A string. Must be a constant. - * @param replacement - * replacement string. A string. Must be a constant. - * @param pos - * position to start replacement. The first position is 1. An integer. Must be a constant. - * @group string_funcs - * @since 4.3.0 + * The column to return some value from. A column of any type. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = - regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) + def any_value(e: Column): Column = Column.fn("any_value", e) /** - * Replace all substrings of the specified string value that match regexp with rep. + * Aggregate function: returns some value of `e` for a group of rows. If `ignoreNulls` is true, + * returns only non-null values. * * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A column that evaluates to a string. - * @param replacement - * replacement string. A column that evaluates to a string. - * @group string_funcs - * @since 2.1.0 + * The column to return some value from. A column of any type. + * @param ignoreNulls + * If true, returns only non-null values. A column that evaluates to a boolean. Must be a + * constant. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = - Column.fn("regexp_replace", e, pattern, replacement) + def any_value(e: Column, ignoreNulls: Column): Column = + Column.fn("any_value", e, ignoreNulls) /** - * Replace all substrings of the specified string value that match regexp with rep, starting at - * the specified position `pos`. + * Aggregate function: returns the number of `TRUE` values for the expression. * * @param e - * target column to work on. A column that evaluates to a string. - * @param pattern - * regex pattern to apply. A column that evaluates to a string. - * @param replacement - * replacement string. A column that evaluates to a string. - * @param pos - * position to start replacement. The first position is 1. A column that evaluates to an - * integer. - * @group string_funcs - * @since 4.3.0 + * The expression to count TRUE values of. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = - Column.fn("regexp_replace", e, pattern, replacement, pos) + def count_if(e: Column): Column = Column.fn("count_if", e) /** - * Returns the substring that matches the regular expression `regexp` within the string `str`. - * If the regular expression is not found, the result is null. + * Returns the current time at the start of query evaluation. Note that the result will contain + * 6 fractional digits of seconds. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * A time. Returns a column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 */ - def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) + def current_time(): Column = { + Column.fn("current_time") + } /** - * Searches a string for a regular expression and returns an integer that indicates the - * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match - * is found, returns 0. + * Returns the current time at the start of query evaluation. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @param precision + * An integer literal in the range [0..6], indicating how many fractional digits of seconds to + * include in the result. A column that evaluates to an integer. Must be a constant. * @return - * Returns a column that evaluates to an integer. + * A time. Returns a column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 */ - def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) + def current_time(precision: Int): Column = { + Column.fn("current_time", lit(precision)) + } /** - * Searches a string for a regular expression and returns an integer that indicates the - * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match - * is found, returns 0. + * Aggregate function: computes a histogram on numeric 'expr' using nb bins. The return value is + * an array of (x,y) pairs representing the centers of the histogram's bins. As the value of + * 'nb' is increased, the histogram approximation gets finer-grained, but may yield artifacts + * around outliers. In practice, 20-40 histogram bins appear to work well, with more bins being + * required for skewed or smaller datasets. Note that this function creates a histogram with + * non-uniform bin widths. It offers no guarantees in terms of the mean-squared-error of the + * histogram, but in practice is comparable to the histograms produced by the R/S-Plus + * statistical computing packages. Note: the output type of the 'x' field in the return value is + * propagated from the input value consumed in the aggregate function. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param regexp - * regex pattern to apply. A column that evaluates to a string. - * @param idx - * matched group id. A column that evaluates to an integer. - * @group string_funcs + * @param e + * The column to compute the histogram on. A column that evaluates to a numeric. + * @param nBins + * The number of histogram bins. A column that evaluates to an integral. Must be a constant. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to an array. */ - def regexp_instr(str: Column, regexp: Column, idx: Column): Column = - Column.fn("regexp_instr", str, regexp, idx) + def histogram_numeric(e: Column, nBins: Column): Column = + Column.fn("histogram_numeric", e, nBins) /** - * Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse - * of base64. + * Aggregate function: returns true if all values of `e` are true. * * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def unbase64(e: Column): Column = Column.fn("unbase64", e) + def every(e: Column): Column = Column.fn("every", e) /** - * Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary column. This is - * the reverse of to_base32. + * Aggregate function: returns true if all values of `e` are true. * * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 4.3.0 + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def from_base32(e: Column): Column = Column.fn("from_base32", e) + def bool_and(e: Column): Column = Column.fn("bool_and", e) /** - * Right-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. + * Aggregate function: returns true if at least one value of `e` is true. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param len - * length of the final string. An integer. Must be a constant. - * @param pad - * chars to append. A string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * @param e + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) + def some(e: Column): Column = Column.fn("some", e) /** - * Right-pad the binary column with pad to a byte length of len. If the binary column is longer - * than len, the return value is shortened to len bytes. + * Aggregate function: returns true if at least one value of `e` is true. * - * @param str - * target column to work on. A column that evaluates to a binary. - * @param len - * byte length of the final binary. An integer. Must be a constant. - * @param pad - * bytes to append. A binary. Must be a constant. - * @group string_funcs - * @since 3.3.0 + * @param e + * The expression to evaluate. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) + def any(e: Column): Column = Column.fn("any", e) /** - * Right-pad the string column with pad to a length of len. If the string column is longer than - * len, the return value is shortened to len characters. + * Aggregate function: returns true if at least one value of `e` is true. * - * @param str - * target column to work on. A column that evaluates to a string or binary. - * @param len - * length of the final result. A column that evaluates to an integer. - * @param pad - * chars or bytes to append. A column that evaluates to a string or binary. - * @group string_funcs - * @since 4.0.0 + * @param e + * column to check if at least one value is true. A column that evaluates to a boolean. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) + def bool_or(e: Column): Column = Column.fn("bool_or", e) /** - * Repeats a string column n times, and returns it as a new string column. + * Aggregate function: returns the bitwise AND of all non-null input values, or null if none. * - * @param str - * target column to work on. A column that evaluates to a string. - * @param n - * number of times to repeat value. A column that evaluates to an integral. Must be a - * constant. - * @group string_funcs - * @since 1.5.0 + * @param e + * target column to compute on. A column that evaluates to an integral. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) + def bit_and(e: Column): Column = Column.fn("bit_and", e) /** - * Repeats a string column n times, and returns it as a new string column. - * - * @param str - * target column to work on. A column that evaluates to a string. - * @param n - * number of times to repeat value. A column that evaluates to an integral. - * @group string_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a string. - */ - def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) - - /** - * Trim the spaces from right end for the specified string value. + * Aggregate function: returns the bitwise OR of all non-null input values, or null if none. * * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * target column to compute on. A column that evaluates to an integral. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def rtrim(e: Column): Column = Column.fn("rtrim", e) + def bit_or(e: Column): Column = Column.fn("bit_or", e) /** - * Trim the specified character string from right end for the specified string column. + * Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. + * * @param e - * target column to work on. A column that evaluates to a string. - * @param trimString - * the trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 2.3.0 + * target column to compute on. A column that evaluates to an integral. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) + def bit_xor(e: Column): Column = Column.fn("bit_xor", e) - /** - * Trim the specified character string from right end for the specified string column. - * @param e - * target column to work on. A column that evaluates to a string. - * @param trim - * the trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a string. - */ - def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) + ////////////////////////////////////////////////////////////////////////////////////////////// + // Window functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Returns the soundex code for the specified expression. + * Window function: computes the differences between consecutive cumulative counter values in a + * time series, thereby converting the counter from the cumulative to the delta format. + * + * Gracefully handles counter resets by returning NULL. Counter resets are detected when the + * counter value decreases. + * + * Use the PARTITION BY clause of the window to separate independent counters. This is done by + * specifying all columns which uniquely identify a time series. These are typically the counter + * name and any attributes tied to the counter. + * + * Use the ORDER BY clause of the window to order the observations by the associated timestamp + * in ascending order. + * + * @param value + * A cumulative counter. Must be a numeric data type. Must be non-negative. * - * @param e - * target column to work on. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 * @return - * Returns a column that evaluates to a string. + * The difference between the current and previous counter value within the window partition, + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. + * @group window_funcs + * @since 4.3.0 */ - def soundex(e: Column): Column = Column.fn("soundex", e) + def counter_diff(value: Column): Column = Column.fn("counter_diff", value) /** - * Splits str around matches of the given pattern. + * Window function: computes the differences between consecutive cumulative counter values in a + * time series, thereby converting the counter from the cumulative to the delta format. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a string representing a regular expression. The regex string should be a Java regular - * expression. A column that evaluates to a string. + * Gracefully handles counter resets by returning NULL. Counter resets are detected when the + * counter value decreases, or when the start time advances between rows. + * + * Use the PARTITION BY clause of the window to separate independent counters. This is done by + * specifying all columns which uniquely identify a time series. These are typically the counter + * name and any attributes tied to the counter. + * + * Use the ORDER BY clause of the window to order the observations by the associated timestamp + * in ascending order. + * + * @param value + * A cumulative counter. Must be a numeric data type. Must be non-negative. + * + * @param startTime + * A timestamp indicating when the counter was last set to zero. Used to signal counter + * resets. * - * @group string_funcs - * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * The difference between the current and previous counter value within the window partition, + * according to the order defined by the window's ORDER BY clause. Returns a column of the + * same type as the input. + * @group window_funcs + * @since 4.3.0 */ - def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) + def counter_diff(value: Column, startTime: Column): Column = + Column.fn("counter_diff", value, startTime) /** - * Splits str around matches of the given pattern. + * Window function: returns the cumulative distribution of values within a window partition, + * i.e. the fraction of rows that are below the current row. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a column of string representing a regular expression. The regex string should be a Java - * regular expression. A column that evaluates to a string. + * {{{ + * N = total number of rows in the partition + * cumeDist(x) = number of values before (and including) x / N + * }}} * - * @group string_funcs - * @since 4.0.0 + * @group window_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a double. */ - def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) + def cume_dist(): Column = Column.fn("cume_dist") /** - * Splits str around matches of the given pattern. + * Window function: returns the rank of rows within a window partition, without any gaps. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a string representing a regular expression. The regex string should be a Java regular - * expression. A column that evaluates to a string. - * @param limit - * an integer expression which controls the number of times the regex is applied.
    - *
  • limit greater than 0: The resulting array's length will not be more than limit, and the - * resulting array's last entry will contain all input beyond the last matched regex.
    • - *
  • limit less than or equal to 0: `regex` will be applied as many times as possible, and - * the resulting array can be of any size.
A column that evaluates to an integer. + * The difference between rank and dense_rank is that denseRank leaves no gaps in ranking + * sequence when there are ties. That is, if you were ranking a competition using dense_rank and + * had three people tie for second place, you would say that all three were in second place and + * that the next person came in third. Rank would give me sequential numbers, making the person + * that came in third place (after the ties) would register as coming in fifth. * - * @group string_funcs - * @since 3.0.0 + * This is equivalent to the DENSE_RANK function in SQL. + * + * @group window_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def split(str: Column, pattern: String, limit: Int): Column = - Column.fn("split", str, lit(pattern), lit(limit)) + def dense_rank(): Column = Column.fn("dense_rank") /** - * Splits str around matches of the given pattern. + * Window function: returns the value that is `offset` rows before the current row, and `null` + * if there is less than `offset` rows before the current row. For example, an `offset` of one + * will return the previous row at any given point in the window partition. * - * @param str - * a string expression to split. A column that evaluates to a string. - * @param pattern - * a column of string representing a regular expression. The regex string should be a Java - * regular expression. A column that evaluates to a string. - * @param limit - * a column of integer expression which controls the number of times the regex is applied. - *
  • limit greater than 0: The resulting array's length will not be more than limit, - * and the resulting array's last entry will contain all input beyond the last matched - * regex.
  • limit less than or equal to 0: `regex` will be applied as many times as - * possible, and the resulting array can be of any size.
A column that evaluates to - * an integer. + * This is equivalent to the LAG function in SQL. * - * @group string_funcs - * @since 4.0.0 + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def split(str: Column, pattern: Column, limit: Column): Column = - Column.fn("split", str, pattern, limit) + def lag(e: Column, offset: Int): Column = lag(e, offset, null) /** - * Substring starts at `pos` and is of length `len` when str is String type or returns the slice - * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type + * Window function: returns the value that is `offset` rows before the current row, and `null` + * if there is less than `offset` rows before the current row. For example, an `offset` of one + * will return the previous row at any given point in the window partition. * - * @param str - * target column to work on. A column that evaluates to a string or binary. - * @param pos - * starting position in str. A column that evaluates to an integral. Must be a constant. - * @param len - * length of chars. A column that evaluates to an integral. Must be a constant. - * @note - * The position is not zero based, but 1 based index. + * This is equivalent to the LAG function in SQL. * - * @group string_funcs - * @since 1.5.0 + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return * Returns a column of the same type as the input. */ - def substring(str: Column, pos: Int, len: Int): Column = - Column.fn("substring", str, lit(pos), lit(len)) + def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) /** - * Substring starts at `pos` and is of length `len` when str is String type or returns the slice - * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type + * Window function: returns the value that is `offset` rows before the current row, and + * `defaultValue` if there is less than `offset` rows before the current row. For example, an + * `offset` of one will return the previous row at any given point in the window partition. * - * @param str - * target column to work on. A column that evaluates to a string or binary. - * @param pos - * starting position in str. A column that evaluates to an integral. - * @param len - * length of chars. A column that evaluates to an integral. - * @note - * The position is not zero based, but 1 based index. + * This is equivalent to the LAG function in SQL. * - * @group string_funcs - * @since 4.0.0 + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 * @return * Returns a column of the same type as the input. */ - def substring(str: Column, pos: Column, len: Column): Column = - Column.fn("substring", str, pos, len) + def lag(columnName: String, offset: Int, defaultValue: Any): Column = { + lag(Column(columnName), offset, defaultValue) + } /** - * Returns the substring from string str before count occurrences of the delimiter delim. If - * count is positive, everything the left of the final delimiter (counting from left) is - * returned. If count is negative, every to the right of the final delimiter (counting from the - * right) is returned. substring_index performs a case-sensitive match when searching for delim. - * - * @param str - * target column to work on. A column that evaluates to a string. - * @param delim - * delimiter of values. A column that evaluates to a string. Must be a constant. - * @param count - * number of occurrences. A column that evaluates to an integral. Must be a constant. - * @group string_funcs - * @since 1.5.0 - * @return - * Returns a column that evaluates to a string. - */ - def substring_index(str: Column, delim: String, count: Int): Column = - Column.fn("substring_index", str, lit(delim), lit(count)) - - /** - * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of - * `src` and proceeding for `len` bytes. + * Window function: returns the value that is `offset` rows before the current row, and + * `defaultValue` if there is less than `offset` rows before the current row. For example, an + * `offset` of one will return the previous row at any given point in the window partition. * - * @param src - * the string that will be replaced. A column that evaluates to a string or binary. - * @param replace - * the substitution string. A column that evaluates to a string or binary. - * @param pos - * the starting position in src. A column that evaluates to an integral. - * @param len - * the number of bytes to replace in src. A column that evaluates to an integral. - * @group string_funcs - * @since 3.0.0 + * This is equivalent to the LAG function in SQL. + * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 * @return * Returns a column of the same type as the input. */ - def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = - Column.fn("overlay", src, replace, pos, len) + def lag(e: Column, offset: Int, defaultValue: Any): Column = { + lag(e, offset, defaultValue, false) + } /** - * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of - * `src`. + * Window function: returns the value that is `offset` rows before the current row, and + * `defaultValue` if there is less than `offset` rows before the current row. `ignoreNulls` + * determines whether null values of row are included in or eliminated from the calculation. For + * example, an `offset` of one will return the previous row at any given point in the window + * partition. * - * @param src - * the string that will be replaced. A column that evaluates to a string or binary. - * @param replace - * the substitution string. A column that evaluates to a string or binary. - * @param pos - * the starting position in src. A column that evaluates to an integral. - * @group string_funcs - * @since 3.0.0 - * @return - * Returns a column that evaluates to a string. - */ - def overlay(src: Column, replace: Column, pos: Column): Column = - Column.fn("overlay", src, replace, pos) - - /** - * Splits a string into arrays of sentences, where each sentence is an array of words. - * @param string - * a string to be split. A column that evaluates to a string. - * @param language - * a language of the locale. A column that evaluates to a string. - * @param country - * a country of the locale. A column that evaluates to a string. - * @group string_funcs + * This is equivalent to the LAG function in SQL. + * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @param ignoreNulls + * whether to ignore null values. A column that evaluates to a boolean. Must be a constant. + * @group window_funcs * @since 3.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def sentences(string: Column, language: Column, country: Column): Column = - Column.fn("sentences", string, language, country) + def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = + Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) /** - * Splits a string into arrays of sentences, where each sentence is an array of words. The - * default `country`('') is used. - * @param string - * a string to be split. A column that evaluates to a string. - * @param language - * a language of the locale. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * Window function: returns the value that is `offset` rows after the current row, and `null` if + * there is less than `offset` rows after the current row. For example, an `offset` of one will + * return the next row at any given point in the window partition. + * + * This is equivalent to the LEAD function in SQL. + * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def sentences(string: Column, language: Column): Column = - Column.fn("sentences", string, language) + def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } /** - * Splits a string into arrays of sentences, where each sentence is an array of words. The - * default locale is used. - * @param string - * a string to be split. A column that evaluates to a string. - * @group string_funcs - * @since 3.2.0 + * Window function: returns the value that is `offset` rows after the current row, and `null` if + * there is less than `offset` rows after the current row. For example, an `offset` of one will + * return the next row at any given point in the window partition. + * + * This is equivalent to the LEAD function in SQL. + * + * @param e + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def sentences(string: Column): Column = Column.fn("sentences", string) + def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } /** - * Translate any character in the src by a character in replaceString. The characters in - * replaceString correspond to the characters in matchingString. The translate will happen when - * any character in the string matches the character in the `matchingString`. + * Window function: returns the value that is `offset` rows after the current row, and + * `defaultValue` if there is less than `offset` rows after the current row. For example, an + * `offset` of one will return the next row at any given point in the window partition. * - * @param src - * source column to work on. A column that evaluates to a string. - * @param matchingString - * matching characters. A column that evaluates to a string. Must be a constant. - * @param replaceString - * characters for replacement. A column that evaluates to a string. Must be a constant. - * @group string_funcs - * @since 1.5.0 + * This is equivalent to the LEAD function in SQL. + * + * @param columnName + * name of column or expression. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def translate(src: Column, matchingString: String, replaceString: String): Column = - Column.fn("translate", src, lit(matchingString), lit(replaceString)) + def lead(columnName: String, offset: Int, defaultValue: Any): Column = { + lead(Column(columnName), offset, defaultValue) + } /** - * Trim the spaces from both ends for the specified string column. + * Window function: returns the value that is `offset` rows after the current row, and + * `defaultValue` if there is less than `offset` rows after the current row. For example, an + * `offset` of one will return the next row at any given point in the window partition. + * + * This is equivalent to the LEAD function in SQL. * * @param e - * The string column to trim. A column that evaluates to a string. - * @group string_funcs - * @since 1.5.0 + * the column to compute on. A column of any type. + * @param offset + * number of rows to extend. A column that evaluates to an integer. Must be a constant. + * @param defaultValue + * default value. A column of any type. + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def trim(e: Column): Column = Column.fn("trim", e) + def lead(e: Column, offset: Int, defaultValue: Any): Column = { + lead(e, offset, defaultValue, false) + } /** - * Trim the specified character from both ends for the specified string column. + * Window function: returns the value that is `offset` rows after the current row, and + * `defaultValue` if there is less than `offset` rows after the current row. `ignoreNulls` + * determines whether null values of row are included in or eliminated from the calculation. The + * default value of `ignoreNulls` is false. For example, an `offset` of one will return the next + * row at any given point in the window partition. + * + * This is equivalent to the LEAD function in SQL. + * * @param e - * The string column to trim. A column that evaluates to a string. - * @param trimString - * The trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 2.3.0 + * The column to compute the lead value for. A column of any type. + * @param offset + * Number of rows after the current row to look ahead. A column that evaluates to an integral. + * Must be a constant. + * @param defaultValue + * Value to return when there are fewer than `offset` rows after the current row. A column of + * any type. Must be a constant. + * @param ignoreNulls + * Whether to skip null values when computing the result. A column that evaluates to a + * boolean. Must be a constant. + * @group window_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) + def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = + Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) /** - * Trim the specified character from both ends for the specified string column. + * Window function: returns the value that is the `offset`th row of the window frame (counting + * from 1), and `null` if the size of window frame is less than `offset` rows. + * + * It will return the `offset`th non-null value it sees when ignoreNulls is set to true. If all + * values are null, then null is returned. + * + * This is equivalent to the nth_value function in SQL. + * * @param e - * The string column to trim. A column that evaluates to a string. - * @param trim - * The trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 4.0.0 + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. + * @param ignoreNulls + * Whether the nth value should skip nulls when determining which row to use. A column that + * evaluates to a boolean. Must be a constant. + * @group window_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) + def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = + Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) /** - * Converts a string column to upper case. + * Window function: returns the value that is the `offset`th row of the window frame (counting + * from 1), and `null` if the size of window frame is less than `offset` rows. + * + * This is equivalent to the nth_value function in SQL. * * @param e - * The input column to convert to upper case. A column that evaluates to a string. - * @group string_funcs - * @since 1.3.0 + * The column to extract the value from. A column of any type. + * @param offset + * The 1-based row number within the window frame to use as the value. A column that evaluates + * to an integral. Must be a constant. + * @group window_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def upper(e: Column): Column = Column.fn("upper", e) + def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) /** - * Converts the input `e` to a binary value based on the supplied `format`. The `format` can be - * a case-insensitive string literal of "hex", "utf-8", "utf8", or "base64". By default, the - * binary format for conversion is "hex" if `format` is omitted. The function returns NULL if at - * least one of the input parameters is NULL. + * Window function: returns the ntile group id (from 1 to `n` inclusive) in an ordered window + * partition. For example, if `n` is 4, the first quarter of the rows will get value 1, the + * second quarter will get 2, the third quarter will get 3, and the last quarter will get 4. * - * @param e - * The input value to convert. A column that evaluates to a string. - * @param f - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant. - * @group string_funcs - * @since 3.5.0 + * This is equivalent to the NTILE function in SQL. + * + * @param n + * The number of groups to divide the window partition into. A column that evaluates to an + * integral. Must be a constant. + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an integer. */ - def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) + def ntile(n: Int): Column = Column.fn("ntile", lit(n)) /** - * Converts the input `e` to a binary value based on the default format "hex". The function - * returns NULL if at least one of the input parameters is NULL. + * Window function: returns the relative rank (i.e. percentile) of rows within a window + * partition. * - * @param e - * The input value to convert. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * This is computed by: + * {{{ + * (rank of row in its partition - 1) / (number of rows in the partition - 1) + * }}} + * + * This is equivalent to the PERCENT_RANK function in SQL. + * + * @group window_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def to_binary(e: Column): Column = Column.fn("to_binary", e) + def percent_rank(): Column = Column.fn("percent_rank") - // scalastyle:off line.size.limit /** - * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. - * The format can consist of the following characters, case insensitive: '0' or '9': Specifies - * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a - * sequence of digits in the input value, generating a result string of the same length as the - * corresponding sequence in the format string. The result string is left-padded with zeros if - * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts - * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': - * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': - * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to - * the left and right of each grouping separator. '$': Specifies the location of the $ currency - * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' - * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note - * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the - * end of the format string; specifies that the result string will be wrapped by angle brackets - * if the input value is negative. + * Window function: returns the rank of rows within a window partition. * - * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime - * Patterns. If `e` is a binary, it is converted to a string in one of the formats: - * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input - * binary is decoded to UTF-8 string. + * The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking + * sequence when there are ties. That is, if you were ranking a competition using dense_rank and + * had three people tie for second place, you would say that all three were in second place and + * that the next person came in third. Rank would give me sequential numbers, making the person + * that came in third place (after the ties) would register as coming in fifth. * - * @param e - * The input value to convert. A column that evaluates to a numeric, date, timestamp or - * binary. - * @param format - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant when `e` is a numeric or binary value. - * @group string_funcs - * @since 3.5.0 + * This is equivalent to the RANK function in SQL. + * + * @group window_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an integer. */ - // scalastyle:on line.size.limit - def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) + def rank(): Column = Column.fn("rank") - // scalastyle:off line.size.limit /** - * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. - * The format can consist of the following characters, case insensitive: '0' or '9': Specifies - * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a - * sequence of digits in the input value, generating a result string of the same length as the - * corresponding sequence in the format string. The result string is left-padded with zeros if - * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts - * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': - * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': - * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to - * the left and right of each grouping separator. '$': Specifies the location of the $ currency - * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' - * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note - * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the - * end of the format string; specifies that the result string will be wrapped by angle brackets - * if the input value is negative. - * - * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime - * Patterns. If `e` is a binary, it is converted to a string in one of the formats: - * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input - * binary is decoded to UTF-8 string. + * Window function: returns a sequential number starting at 1 within a window partition. * - * @param e - * The input value to convert. A column that evaluates to a numeric, date, timestamp or - * binary. - * @param format - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant when `e` is a numeric or binary value. - * @group string_funcs - * @since 3.5.0 + * @group window_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an integer. */ - // scalastyle:on line.size.limit - def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) + def row_number(): Column = Column.fn("row_number") + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Non-aggregate functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Convert string 'e' to a number based on the string format 'format'. Throws an exception if - * the conversion fails. The format can consist of the following characters, case insensitive: - * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format - * string matches a sequence of digits in the input string. If the 0/9 sequence starts with 0 - * and is before the decimal point, it can only match a digit sequence of the same size. - * Otherwise, if the sequence starts with 9 or is after the decimal point, it can match a digit - * sequence that has the same or smaller size. '.' or 'D': Specifies the position of the decimal - * point (optional, only allowed once). ',' or 'G': Specifies the position of the grouping - * (thousands) separator (,). There must be a 0 or 9 to the left and right of each grouping - * separator. 'expr' must match the grouping separator relevant for the size of the number. '$': - * Specifies the location of the $ currency sign. This character may only be specified once. 'S' - * or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at the - * beginning or end of the format string). Note that 'S' allows '-' but 'MI' does not. 'PR': - * Only allowed at the end of the format string; specifies that 'expr' indicates a negative - * number with wrapping angled brackets. + * Creates a new array column. The input columns must all have the same data type. * - * @param e - * The input string to convert to a number. A column that evaluates to a string. - * @param format - * The format to use to convert the value. A column that evaluates to a string. Must be a - * constant. - * @group string_funcs - * @since 3.5.0 + * @param cols + * The columns to combine into an array. Each is a column of any type, and all must share the + * same data type. + * @group array_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column that evaluates to an array. */ - def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) + @scala.annotation.varargs + def array(cols: Column*): Column = Column.fn("array", cols: _*) /** - * Replaces all occurrences of `search` with `replace`. - * - * @param src - * A column of strings to be replaced. A column that evaluates to a string. - * @param search - * A column of strings. If `search` is not found in `str`, `str` is returned unchanged. A - * column that evaluates to a string. - * @param replace - * A column of strings. If `replace` is not specified or is an empty string, nothing replaces - * the string that is removed from `str`. A column that evaluates to a string. + * Creates a new array column. The input columns must all have the same data type. * - * @group string_funcs - * @since 3.5.0 + * @group array_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def replace(src: Column, search: Column, replace: Column): Column = - Column.fn("replace", src, search, replace) + @scala.annotation.varargs + def array(colName: String, colNames: String*): Column = { + array((colName +: colNames).map(col): _*) + } /** - * Replaces all occurrences of `search` with `replace`. - * - * @param src - * A column of strings to be replaced. A column that evaluates to a string. - * @param search - * A column of strings. If `search` is not found in `src`, `src` is returned unchanged. A - * column that evaluates to a string. + * Creates a new map column. The input columns must be grouped as key-value pairs, e.g. (key1, + * value1, key2, value2, ...). The key columns must all have the same data type, and can't be + * null. The value columns must all have the same data type. * - * @group string_funcs - * @since 3.5.0 + * @param cols + * The columns grouped as key-value pairs (key1, value1, key2, value2, ...). Each is a column + * of any type; key columns must share a type and value columns must share a type. + * @group map_funcs + * @since 2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) + @scala.annotation.varargs + def map(cols: Column*): Column = Column.fn("map", cols: _*) /** - * Splits `str` by delimiter and return requested part of the split (1-based). If any input is - * null, returns null. if `partNum` is out of range of split parts, returns empty string. If - * `partNum` is 0, throws an error. If `partNum` is negative, the parts are counted backward - * from the end of the string. If the `delimiter` is an empty string, the `str` is not split. + * Creates a struct with the given field names and values. * - * @param str - * A column of strings to be split. A column that evaluates to a string. - * @param delimiter - * The delimiter used for split. A column that evaluates to a string. - * @param partNum - * The requested part of the split (1-based). A column that evaluates to an integral. - * @group string_funcs + * @param cols + * The field names and values grouped as pairs (name1, value1, name2, value2, ...). Names are + * columns that evaluate to a string; values are columns of any type. + * @group struct_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a struct. */ - def split_part(str: Column, delimiter: Column, partNum: Column): Column = - Column.fn("split_part", str, delimiter, partNum) + @scala.annotation.varargs + def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) /** - * Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of - * byte array that starts at `pos` and is of length `len`. + * Creates a new map column. The array in the first column is used for keys. The array in the + * second column is used for values. All elements in the array for key should not be null. * - * @param str - * The input from which to take the substring. A column that evaluates to a string or binary. - * @param pos - * The starting position of the substring. A column that evaluates to an integral. - * @param len - * The length of the substring. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * @param keys + * The array of keys for the map; elements must not be null. A column that evaluates to an + * array. + * @param values + * The array of values for the map. A column that evaluates to an array. + * @group map_funcs + * @since 2.4 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a map. */ - def substr(str: Column, pos: Column, len: Column): Column = - Column.fn("substr", str, pos, len) + def map_from_arrays(keys: Column, values: Column): Column = + Column.fn("map_from_arrays", keys, values) /** - * Returns the substring of `str` that starts at `pos`, or the slice of byte array that starts - * at `pos`. + * Creates a map after splitting the text into key/value pairs using delimiters. Both + * `pairDelim` and `keyValueDelim` are treated as regular expressions. * - * @param str - * The input from which to take the substring. A column that evaluates to a string or binary. - * @param pos - * The starting position of the substring. A column that evaluates to an integral. - * @group string_funcs + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. + * @param keyValueDelim + * Delimiter used to split key and value, treated as a regular expression. A column that + * evaluates to a string. + * @group map_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a map. */ - def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) + def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = + Column.fn("str_to_map", text, pairDelim, keyValueDelim) /** - * Formats the arguments in printf-style and returns the result as a string column. + * Creates a map after splitting the text into key/value pairs using delimiters. The `pairDelim` + * is treated as regular expressions. * - * @param format - * A format string that can contain embedded format tags. A column that evaluates to a string. - * @param arguments - * The values to be used in formatting. Columns that evaluate to any type. - * @group string_funcs + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @param pairDelim + * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to + * a string. + * @group map_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - @scala.annotation.varargs - def printf(format: Column, arguments: Column*): Column = - Column.fn("printf", (format +: arguments): _*) + def str_to_map(text: Column, pairDelim: Column): Column = + Column.fn("str_to_map", text, pairDelim) /** - * Returns the position of the first occurrence of `substr` in `str` after position `start`. The - * given `start` and return value are 1-based. + * Creates a map after splitting the text into key/value pairs using delimiters. * - * @param substr - * The substring to search for. A column that evaluates to a string. - * @param str - * The string to search in. A column that evaluates to a string. - * @param start - * The 1-based position to start the search from. A column that evaluates to an integral. - * @group string_funcs + * @param text + * The text to split into key/value pairs. A column that evaluates to a string. + * @group map_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a map. */ - def position(substr: Column, str: Column, start: Column): Column = - Column.fn("position", substr, str, start) + def str_to_map(text: Column): Column = Column.fn("str_to_map", text) /** - * Returns the position of the first occurrence of `substr` in `str` after position `1`. The - * return value are 1-based. + * Marks a DataFrame as small enough for use in broadcast joins. * - * @param substr - * The substring to search for. A column that evaluates to a string. - * @param str - * The string to search in. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * The following example marks the right DataFrame for broadcast hash join using `joinKey`. + * {{{ + * // left and right are DataFrames + * left.join(broadcast(right), "joinKey") + * }}} + * + * @group normal_funcs + * @since 1.5.0 + */ + def broadcast[U](df: Dataset[U]): df.type = { + df.hint("broadcast").asInstanceOf[df.type] + } + + /** + * Returns the first column that is not null, or null if all inputs are null. + * + * For example, `coalesce(a, b, c)` will return a if a is not null, or b if a is null and b is + * not null, or c if both a and b are null but c is not null. + * + * @param e + * the columns to work on. A column that evaluates to any type. + * @group conditional_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def position(substr: Column, str: Column): Column = - Column.fn("position", substr, str) + @scala.annotation.varargs + def coalesce(e: Column*): Column = Column.fn("coalesce", e: _*) /** - * Returns a boolean. The value is True if str ends with suffix. Returns NULL if either input - * expression is NULL. Otherwise, returns False. Both str or suffix must be of STRING or BINARY - * type. + * Creates a string column for the file name of the current Spark task. * - * @param str - * The string to test. A column that evaluates to a string or binary. - * @param suffix - * The suffix to test for. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @group misc_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def endswith(str: Column, suffix: Column): Column = - Column.fn("endswith", str, suffix) + def input_file_name(): Column = Column.fn("input_file_name") /** - * Returns a boolean. The value is True if str starts with prefix. Returns NULL if either input - * expression is NULL. Otherwise, returns False. Both str or prefix must be of STRING or BINARY - * type. + * Return true iff the column is NaN. * - * @param str - * The string to test. A column that evaluates to a string or binary. - * @param prefix - * The prefix to test for. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @param e + * the column to check. A column that evaluates to a numeric. + * @group predicate_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a boolean. */ - def startswith(str: Column, prefix: Column): Column = - Column.fn("startswith", str, prefix) + def isnan(e: Column): Column = e.isNaN /** - * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the - * result is equivalent to char(n % 256) + * Return true iff the column is null. * - * @param n - * The code point value. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * @param e + * the column to check. A column that evaluates to any type. + * @group predicate_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a boolean. */ - def char(n: Column): Column = Column.fn("char", n) + def isnull(e: Column): Column = e.isNull /** - * Removes the leading and trailing space characters from `str`. + * A column expression that generates monotonically increasing 64-bit integers. * - * @param str - * The string to trim. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * The generated ID is guaranteed to be monotonically increasing and unique, but not + * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the + * record number within each partition in the lower 33 bits. The assumption is that the data + * frame has less than 1 billion partitions, and each partition has less than 8 billion records. + * + * As an example, consider a `DataFrame` with two partitions, each with 3 records. This + * expression would return the following IDs: + * + * {{{ + * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. + * }}} + * + * @group misc_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def btrim(str: Column): Column = Column.fn("btrim", str) + @deprecated("Use monotonically_increasing_id()", "2.0.0") + def monotonicallyIncreasingId(): Column = monotonically_increasing_id() /** - * Remove the leading and trailing `trim` characters from `str`. + * A column expression that generates monotonically increasing 64-bit integers. * - * @param str - * The string to trim. A column that evaluates to a string. - * @param trim - * The trim string characters to trim. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * The generated ID is guaranteed to be monotonically increasing and unique, but not + * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the + * record number within each partition in the lower 33 bits. The assumption is that the data + * frame has less than 1 billion partitions, and each partition has less than 8 billion records. + * + * As an example, consider a `DataFrame` with two partitions, each with 3 records. This + * expression would return the following IDs: + * + * {{{ + * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. + * }}} + * + * @group misc_funcs + * @since 1.6.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) + def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") /** - * This is a special version of `to_binary` that performs the same operation, but returns a NULL - * value instead of raising an error if the conversion cannot be performed. + * Returns col1 if it is not NaN, or col2 if col1 is NaN. * - * @param e - * The string to convert. A column that evaluates to a string. - * @param f - * The format to use for the conversion. A column that evaluates to a string. Must be a - * constant. - * @group string_funcs - * @since 3.5.0 + * Both inputs should be floating point columns (DoubleType or FloatType). + * + * @param col1 + * the first column to check. A column that evaluates to a numeric. + * @param col2 + * the column to return if the first is NaN. A column that evaluates to a numeric. + * @group conditional_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the first input. */ - def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) + def nanvl(col1: Column, col2: Column): Column = Column.fn("nanvl", col1, col2) /** - * This is a special version of `to_binary` that performs the same operation, but returns a NULL - * value instead of raising an error if the conversion cannot be performed. + * Unary minus, i.e. negate the expression. + * {{{ + * // Select the amount column and negates all values. + * // Scala: + * df.select( -df("amount") ) + * + * // Java: + * df.select( negate(df.col("amount")) ); + * }}} * * @param e - * The string to convert. A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * the column to negate. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) + def negate(e: Column): Column = -e /** - * Convert string `e` to a number based on the string format `format`. Returns NULL if the - * string `e` does not match the expected format. The format follows the same semantics as the - * to_number function. + * Inversion of boolean expression, i.e. NOT. + * {{{ + * // Scala: select rows that are not active (isActive === false) + * df.filter( !df("isActive") ) + * + * // Java: + * df.filter( not(df.col("isActive")) ); + * }}} * * @param e - * The string to convert. A column that evaluates to a string. - * @param format - * The format used to convert the string to a number. A column that evaluates to a string. - * Must be a constant. - * @group string_funcs - * @since 3.5.0 + * the column to invert. A column that evaluates to a boolean. + * @group predicate_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column that evaluates to a boolean. */ - def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) + def not(e: Column): Column = !e /** - * Returns the character length of string data or number of bytes of binary data. The length of - * string data includes the trailing spaces. The length of binary data includes binary zeros. + * Generate a random column with independent and identically distributed (i.i.d.) samples + * uniformly distributed in [0.0, 1.0). * - * @param str - * Input column or strings. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @param seed + * the seed for the random generator. + * @note + * The function is non-deterministic in general case. + * + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def char_length(str: Column): Column = Column.fn("char_length", str) + def rand(seed: Long): Column = Column.fn("rand", lit(seed)) /** - * Returns the character length of string data or number of bytes of binary data. The length of - * string data includes the trailing spaces. The length of binary data includes binary zeros. + * Generate a random column with independent and identically distributed (i.i.d.) samples + * uniformly distributed in [0.0, 1.0). * - * @param str - * Input column or strings. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @note + * The function is non-deterministic in general case. + * + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def character_length(str: Column): Column = Column.fn("character_length", str) + def rand(): Column = rand(SparkClassUtils.random.nextLong) /** - * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the - * result is equivalent to chr(n % 256) + * Generate a column with independent and identically distributed (i.i.d.) samples from the + * standard normal distribution. * - * @param n - * The code point. A column that evaluates to an integral. - * @group string_funcs - * @since 3.5.0 + * @param seed + * the seed for the random generator. + * @note + * The function is non-deterministic in general case. + * + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def chr(n: Column): Column = Column.fn("chr", n) + def randn(seed: Long): Column = Column.fn("randn", lit(seed)) /** - * Returns a boolean. The value is True if right is found inside left. Returns NULL if either - * input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or - * BINARY type. + * Generate a column with independent and identically distributed (i.i.d.) samples from the + * standard normal distribution. * - * @param left - * The input to check, may be NULL. A column that evaluates to a string or binary. - * @param right - * The input to find, may be NULL. A column that evaluates to a string or binary. - * @group string_funcs - * @since 3.5.0 + * @note + * The function is non-deterministic in general case. + * + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a double. */ - def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) + def randn(): Column = randn(SparkClassUtils.random.nextLong) /** - * Returns the `n`-th input, e.g., returns `input2` when `n` is 2. The function returns NULL if - * the index exceeds the length of the array and `spark.sql.ansi.enabled` is set to false. If - * `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException for invalid - * indices. + * Returns a string of the specified length whose characters are chosen uniformly at random from + * the following pool of characters: 0-9, a-z, A-Z. The string length must be a constant + * two-byte or four-byte integer (SMALLINT or INT, respectively). * - * @param inputs - * The index followed by the inputs to select from. Columns where the first evaluates to an - * integral and the rest evaluate to strings or binaries. + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. * @group string_funcs - * @since 3.5.0 + * @since 4.0.0 * @return * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) + def randstr(length: Column): Column = + randstr(length, lit(SparkClassUtils.random.nextLong)) /** - * Returns the index (1-based) of the given string (`str`) in the comma-delimited list - * (`strArray`). Returns 0, if the string was not found or if the given string (`str`) contains - * a comma. + * Returns a string of the specified length whose characters are chosen uniformly at random from + * the following pool of characters: 0-9, a-z, A-Z, with the chosen random seed. The string + * length must be a constant two-byte or four-byte integer (SMALLINT or INT, respectively). * - * @param str - * The given string to be found. A column that evaluates to a string. - * @param strArray - * The comma-delimited list. A column that evaluates to a string. + * @param length + * the number of characters in the string to generate. A column that evaluates to an integral. + * Must be a constant. + * @param seed + * the random seed to use. A column that evaluates to an integral. * @group string_funcs - * @since 3.5.0 + * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. + */ + def randstr(length: Column, seed: Column): Column = Column.fn("randstr", length, seed) + + /** + * Partition ID. + * + * @note + * This is non-deterministic because it depends on data partitioning and task scheduling. + * + * @group misc_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to an integer. */ - def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) + def spark_partition_id(): Column = Column.fn("spark_partition_id") /** - * Returns `str` with all characters changed to lowercase. + * Computes the square root of the specified float value. * - * @param str - * A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @param e + * the value to compute the square root of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def lcase(str: Column): Column = Column.fn("lcase", str) + def sqrt(e: Column): Column = Column.fn("sqrt", e) /** - * Returns `str` with all characters changed to uppercase. + * Computes the square root of the specified float value. * - * @param str - * A column that evaluates to a string. - * @group string_funcs - * @since 3.5.0 + * @param colName + * the name of a numeric column to compute the square root of. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def ucase(str: Column): Column = Column.fn("ucase", str) + def sqrt(colName: String): Column = sqrt(Column(colName)) /** - * Returns the leftmost `len`(`len` can be string type) characters from the string `str`, if - * `len` is less or equal than 0 the result is an empty string. + * Returns the sum of `left` and `right` and the result is null on overflow. The acceptable + * input types are the same with the `+` operator. * - * @param str - * Input column or strings. A column that evaluates to a string or binary. - * @param len - * The number of leftmost characters. A column that evaluates to an integral. - * @group string_funcs + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. + * @group math_funcs * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - def left(str: Column, len: Column): Column = Column.fn("left", str, len) + def try_add(left: Column, right: Column): Column = Column.fn("try_add", left, right) /** - * Returns the rightmost `len`(`len` can be string type) characters from the string `str`, if - * `len` is less or equal than 0 the result is an empty string. + * Returns the mean calculated from values of a group and the result is null on overflow. * - * @param str - * Input column or strings. A column that evaluates to a string. - * @param len - * The number of rightmost characters. A column that evaluates to an integral. - * @group string_funcs + * @param e + * the value to compute the mean of. A column that evaluates to a numeric or interval. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a double. */ - def right(str: Column, len: Column): Column = Column.fn("right", str, len) + def try_avg(e: Column): Column = Column.fn("try_avg", e) /** - * Returns `str` enclosed by single quotes and each instance of single quote in it is preceded - * by a backslash. + * Returns `dividend``/``divisor`. It always performs floating point division. Its result is + * always null if `divisor` is 0. * - * @param str - * A column that evaluates to a string. - * @group string_funcs - * @since 4.1.0 + * @param left + * the dividend. A column that evaluates to a numeric or interval. + * @param right + * the divisor. A column that evaluates to a numeric. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def quote(str: Column): Column = Column.fn("quote", str) + def try_divide(left: Column, right: Column): Column = Column.fn("try_divide", left, right) /** - * Masks the given string value. The function replaces characters with 'X' or 'x', and numbers - * with 'n'. This can be useful for creating copies of tables with sensitive information - * removed. - * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. + * Returns the remainder of `dividend``/``divisor`. Its result is always null if `divisor` is 0. * - * @group string_funcs - * @since 3.5.0 + * @param left + * the dividend. A column that evaluates to a numeric. + * @param right + * the divisor. A column that evaluates to a numeric. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def mask(input: Column): Column = Column.fn("mask", input) + def try_mod(left: Column, right: Column): Column = Column.fn("try_mod", left, right) /** - * Masks the given string value. The function replaces upper-case characters with specific - * character, lower-case characters with 'x', and numbers with 'n'. This can be useful for - * creating copies of tables with sensitive information removed. + * Returns `left``*``right` and the result is null on overflow. The acceptable input types are + * the same with the `*` operator. * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * - * @group string_funcs + * @param left + * the multiplicand. A column that evaluates to a numeric or interval. + * @param right + * the multiplier. A column that evaluates to a numeric or interval. + * @group math_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def mask(input: Column, upperChar: Column): Column = - Column.fn("mask", input, upperChar) + def try_multiply(left: Column, right: Column): Column = Column.fn("try_multiply", left, right) /** - * Masks the given string value. The function replaces upper-case and lower-case characters with - * the characters specified respectively, and numbers with 'n'. This can be useful for creating - * copies of tables with sensitive information removed. - * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param lowerChar - * character to replace lower-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. + * Returns `left``-``right` and the result is null on overflow. The acceptable input types are + * the same with the `-` operator. * - * @group string_funcs + * @param left + * the left operand. A column that evaluates to a numeric or interval. + * @param right + * the right operand. A column that evaluates to a numeric or interval. + * @group math_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def mask(input: Column, upperChar: Column, lowerChar: Column): Column = - Column.fn("mask", input, upperChar, lowerChar) + def try_subtract(left: Column, right: Column): Column = Column.fn("try_subtract", left, right) /** - * Masks the given string value. The function replaces upper-case, lower-case characters and - * numbers with the characters specified respectively. This can be useful for creating copies of - * tables with sensitive information removed. - * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param lowerChar - * character to replace lower-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param digitChar - * character to replace digit characters with. Specify NULL to retain original character. A - * column that evaluates to a string. + * Returns the sum calculated from values of a group and the result is null on overflow. * - * @group string_funcs + * @param e + * the value to compute the sum of. A column that evaluates to a numeric or interval. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a numeric. */ - def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column): Column = - Column.fn("mask", input, upperChar, lowerChar, digitChar) + def try_sum(e: Column): Column = Column.fn("try_sum", e) /** - * Masks the given string value. This can be useful for creating copies of tables with sensitive - * information removed. + * Creates a new struct column. If the input column is a column in a `DataFrame`, or a derived + * column expression that is named (i.e. aliased), its name would be retained as the + * StructField's name, otherwise, the newly generated StructField's name would be auto generated + * as `col` with a suffix `index + 1`, i.e. col1, col2, col3, ... * - * @param input - * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a - * string. - * @param upperChar - * character to replace upper-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param lowerChar - * character to replace lower-case characters with. Specify NULL to retain original character. - * A column that evaluates to a string. - * @param digitChar - * character to replace digit characters with. Specify NULL to retain original character. A - * column that evaluates to a string. - * @param otherChar - * character to replace all other characters with. Specify NULL to retain original character. - * A column that evaluates to a string. + * @param cols + * the columns to contain in the output struct. A column of any type. + * @group struct_funcs + * @since 1.4.0 + * @return + * Returns a column that evaluates to a struct. + */ + @scala.annotation.varargs + def struct(cols: Column*): Column = Column.fn("struct", cols: _*) + + /** + * Creates a new struct column that composes multiple input columns. * - * @group string_funcs - * @since 3.5.0 + * @param colName + * the name of the first column to contain in the output struct. + * @param colNames + * the names of the remaining columns to contain in the output struct. + * @group struct_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a struct. */ - def mask( - input: Column, - upperChar: Column, - lowerChar: Column, - digitChar: Column, - otherChar: Column): Column = - Column.fn("mask", input, upperChar, lowerChar, digitChar, otherChar) + @scala.annotation.varargs + def struct(colName: String, colNames: String*): Column = { + struct((colName +: colNames).map(col): _*) + } - ////////////////////////////////////////////////////////////////////////////////////////////// - // Bitwise Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + /** + * Evaluates a list of conditions and returns one of multiple possible result expressions. If + * otherwise is not defined at the end, null is returned for unmatched conditions. + * + * {{{ + * // Example: encoding gender string column into integer. + * + * // Scala: + * people.select(when(people("gender") === "male", 0) + * .when(people("gender") === "female", 1) + * .otherwise(2)) + * + * // Java: + * people.select(when(col("gender").equalTo("male"), 0) + * .when(col("gender").equalTo("female"), 1) + * .otherwise(2)) + * }}} + * + * @param condition + * the condition to evaluate. A column that evaluates to a boolean. + * @param value + * the value to return when the condition is true. A literal value, or a column expression. + * @group conditional_funcs + * @since 1.4.0 + * @return + * Returns a column of the same type as the input. + */ + def when(condition: Column, value: Any): Column = + Column(internal.CaseWhenOtherwise(Seq(condition.node -> lit(value).node))) /** * Computes bitwise NOT (~) of a number. @@ -4431,2865 +4899,2226 @@ object functions { def getbit(e: Column, pos: Column): Column = Column.fn("getbit", e, pos) /** - * Shift the given value numBits left. If the given value is a long value, this function will - * return a long value else it will return an integer value. + * Parses the expression string into the column that it represents, similar to + * [[Dataset#selectExpr]]. + * {{{ + * // get the number of words of each length + * df.groupBy(expr("length(word)")).count() + * }}} * - * @group bitwise_funcs + * @group normal_funcs * @since 1.5.0 - * @return - * Returns a column of the same type as the input. */ - @deprecated("Use shiftleft", "3.2.0") - def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) + def expr(expr: String): Column = Column(internal.SqlExpression(expr)) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Math Functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Shift the given value numBits left. If the given value is a long value, this function will - * return a long value else it will return an integer value. + * Computes the absolute value of a numeric value. * * @param e - * the value to shift. A column that evaluates to an integral. - * @param numBits - * the number of bits to shift left. A column that evaluates to an integral. Must be a - * constant. - * @group bitwise_funcs - * @since 3.2.0 + * the value to compute the absolute value of. A column that evaluates to a numeric or + * interval. + * @group math_funcs + * @since 1.3.0 * @return * Returns a column of the same type as the input. */ - def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) + def abs(e: Column): Column = Column.fn("abs", e) /** - * (Signed) shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. + * @param e + * the value to compute the inverse cosine of. A column that evaluates to a double. + * @return + * inverse cosine of `e` in radians, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. * - * @group bitwise_funcs - * @since 1.5.0 + * @group math_funcs + * @since 1.4.0 + */ + def acos(e: Column): Column = Column.fn("acos", e) + + /** + * @param columnName + * the value to compute the inverse cosine of. * @return - * Returns a column of the same type as the input. + * inverse cosine of `columnName`, as if computed by `java.lang.Math.acos`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - @deprecated("Use shiftright", "3.2.0") - def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) + def acos(columnName: String): Column = acos(Column(columnName)) /** - * (Signed) shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. - * * @param e - * the value to shift. A column that evaluates to an integral. - * @param numBits - * the number of bits to shift right. A column that evaluates to an integral. Must be a - * constant. - * @group bitwise_funcs - * @since 3.2.0 + * the value to compute the inverse hyperbolic cosine of. A column that evaluates to a double. * @return - * Returns a column of the same type as the input. + * inverse hyperbolic cosine of `e`. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.1.0 */ - def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) + def acosh(e: Column): Column = Column.fn("acosh", e) /** - * Unsigned shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. - * - * @group bitwise_funcs - * @since 1.5.0 + * @param columnName + * the value to compute the inverse hyperbolic cosine of. * @return - * Returns a column of the same type as the input. + * inverse hyperbolic cosine of `columnName`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 3.1.0 */ - @deprecated("Use shiftrightunsigned", "3.2.0") - def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) + def acosh(columnName: String): Column = acosh(Column(columnName)) /** - * Unsigned shift the given value numBits right. If the given value is a long value, it will - * return a long value else it will return an integer value. - * * @param e - * the value to shift. A column that evaluates to an integral. - * @param numBits - * the number of bits to shift right. A column that evaluates to an integral. Must be a - * constant. - * @group bitwise_funcs - * @since 3.2.0 + * the value to compute the inverse sine of. A column that evaluates to a double. * @return - * Returns a column of the same type as the input. + * inverse sine of `e` in radians, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def shiftrightunsigned(e: Column, numBits: Int): Column = - Column.fn("shiftrightunsigned", e, lit(numBits)) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Date and Timestamp Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def asin(e: Column): Column = Column.fn("asin", e) /** - * Create time from hour, minute and second fields. For invalid inputs it will throw an error. - * - * @param hour - * the hour to represent, from 0 to 23. A column that evaluates to an integer. - * @param minute - * the minute to represent, from 0 to 59. A column that evaluates to an integer. - * @param second - * the second to represent, from 0 to 59.999999. A column that evaluates to a decimal. - * @group datetime_funcs - * @since 4.1.0 + * @param columnName + * the value to compute the inverse sine of. * @return - * Returns a column that evaluates to a time. + * inverse sine of `columnName`, as if computed by `java.lang.Math.asin`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def make_time(hour: Column, minute: Column, second: Column): Column = { - Column.fn("make_time", hour, minute, second) - } + def asin(columnName: String): Column = asin(Column(columnName)) /** - * Returns the current time at the start of query evaluation. Note that the result will contain - * 6 fractional digits of seconds. - * + * @param e + * the value to compute the inverse hyperbolic sine of. A column that evaluates to a double. * @return - * A time. Returns a column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * inverse hyperbolic sine of `e`. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.1.0 */ - def current_time(): Column = { - Column.fn("current_time") - } + def asinh(e: Column): Column = Column.fn("asinh", e) /** - * Returns the current time at the start of query evaluation. - * - * @param precision - * An integer literal in the range [0..6], indicating how many fractional digits of seconds to - * include in the result. A column that evaluates to an integer. Must be a constant. + * @param columnName + * the value to compute the inverse hyperbolic sine of. * @return - * A time. Returns a column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * inverse hyperbolic sine of `columnName`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 3.1.0 */ - def current_time(precision: Int): Column = { - Column.fn("current_time", lit(precision)) - } + def asinh(columnName: String): Column = asinh(Column(columnName)) /** - * Returns the date that is `numMonths` after `startDate`. - * - * @param startDate - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param numMonths - * The number of months to add to `startDate`, can be negative to subtract months. A column - * that evaluates to an integer. + * @param e + * the value to compute the inverse tangent of. A column that evaluates to a double. * @return - * A date, or null if `startDate` was a string that could not be cast to a date. Returns a - * column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * inverse tangent of `e` as if computed by `java.lang.Math.atan`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def add_months(startDate: Column, numMonths: Int): Column = - add_months(startDate, lit(numMonths)) + def atan(e: Column): Column = Column.fn("atan", e) /** - * Returns the date that is `numMonths` after `startDate`. - * - * @param startDate - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param numMonths - * A column of the number of months to add to `startDate`, can be negative to subtract months. - * A column that evaluates to an integer. + * @param columnName + * the value to compute the inverse tangent of. * @return - * A date, or null if `startDate` was a string that could not be cast to a date. Returns a - * column that evaluates to a date. - * @group datetime_funcs - * @since 3.0.0 + * inverse tangent of `columnName`, as if computed by `java.lang.Math.atan`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def add_months(startDate: Column, numMonths: Column): Column = - Column.fn("add_months", startDate, numMonths) + def atan(columnName: String): Column = atan(Column(columnName)) /** - * Returns the current date at the start of query evaluation as a date column. All calls of - * current_date within the same query return the same value. - * - * @group datetime_funcs - * @since 3.5.0 + * @param y + * coordinate on y-axis. A column that evaluates to a double. + * @param x + * coordinate on x-axis. A column that evaluates to a double. * @return - * Returns a column that evaluates to a date. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def curdate(): Column = Column.fn("curdate") + def atan2(y: Column, x: Column): Column = Column.fn("atan2", y, x) /** - * Returns the current date at the start of query evaluation as a date column. All calls of - * current_date within the same query return the same value. - * - * @group datetime_funcs - * @since 1.5.0 + * @param y + * coordinate on y-axis + * @param xName + * coordinate on x-axis * @return - * Returns a column that evaluates to a date. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def current_date(): Column = Column.fn("current_date") + def atan2(y: Column, xName: String): Column = atan2(y, Column(xName)) /** - * Returns the current session local timezone. - * - * @group datetime_funcs - * @since 3.5.0 + * @param yName + * coordinate on y-axis + * @param x + * coordinate on x-axis * @return - * Returns a column that evaluates to a string. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def current_timezone(): Column = Column.fn("current_timezone") + def atan2(yName: String, x: Column): Column = atan2(Column(yName), x) /** - * Returns the current timestamp at the start of query evaluation as a timestamp column. All - * calls of current_timestamp within the same query return the same value. - * - * @group datetime_funcs - * @since 1.5.0 + * @param yName + * coordinate on y-axis + * @param xName + * coordinate on x-axis * @return - * Returns a column that evaluates to a timestamp. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def current_timestamp(): Column = Column.fn("current_timestamp") + def atan2(yName: String, xName: String): Column = + atan2(Column(yName), Column(xName)) /** - * Returns the current timestamp at the start of query evaluation. - * - * @group datetime_funcs - * @since 3.5.0 + * @param y + * coordinate on y-axis + * @param xValue + * coordinate on x-axis * @return - * Returns a column that evaluates to a timestamp. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def now(): Column = Column.fn("now") + def atan2(y: Column, xValue: Double): Column = atan2(y, lit(xValue)) /** - * Returns the current timestamp without time zone at the start of query evaluation as a - * timestamp without time zone column. All calls of localtimestamp within the same query return - * the same value. - * - * @group datetime_funcs - * @since 3.3.0 + * @param yName + * coordinate on y-axis + * @param xValue + * coordinate on x-axis * @return - * Returns a column that evaluates to a timestamp. + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def localtimestamp(): Column = Column.fn("localtimestamp") + def atan2(yName: String, xValue: Double): Column = atan2(Column(yName), xValue) /** - * Converts a date/timestamp/string to a value of string in the format specified by the date - * format given by the second argument. - * - * See Datetime - * Patterns for valid date and time format patterns - * - * @param dateExpr - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp or time. - * @param format - * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to - * a string. + * @param yValue + * coordinate on y-axis + * @param x + * coordinate on x-axis * @return - * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns - * a column that evaluates to a string. - * @note - * Use specialized functions like [[year]] whenever possible as they benefit from a - * specialized implementation. - * @throws IllegalArgumentException - * if the `format` pattern is invalid - * @group datetime_funcs - * @since 1.5.0 + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def date_format(dateExpr: Column, format: String): Column = - Column.fn("date_format", dateExpr, lit(format)) + def atan2(yValue: Double, x: Column): Column = atan2(lit(yValue), x) /** - * Returns the date that is `days` days after `start` - * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * The number of days to add to `start`, can be negative to subtract days. A column that - * evaluates to an integer, short, or byte. + * @param yValue + * coordinate on y-axis + * @param xName + * coordinate on x-axis * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * the theta component of the point (r, theta) in polar coordinates that + * corresponds to the point (x, y) in Cartesian coordinates, as if computed by + * `java.lang.Math.atan2`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def date_add(start: Column, days: Int): Column = date_add(start, lit(days)) + def atan2(yValue: Double, xName: String): Column = atan2(yValue, Column(xName)) /** - * Returns the date that is `days` days after `start` - * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * A column of the number of days to add to `start`, can be negative to subtract days. A - * column that evaluates to an integer, short, or byte. + * @param e + * target column to compute on. A column that evaluates to a numeric. * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 3.0.0 + * inverse hyperbolic tangent of `e`. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.1.0 */ - def date_add(start: Column, days: Column): Column = Column.fn("date_add", start, days) + def atanh(e: Column): Column = Column.fn("atanh", e) /** - * Returns the date that is `days` days after `start` - * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * A column of the number of days to add to `start`, can be negative to subtract days. A - * column that evaluates to an integer, short, or byte. + * @param columnName + * target column to compute on. * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 3.5.0 + * inverse hyperbolic tangent of `columnName`. Returns a column that evaluates to a double. + * @group math_funcs + * @since 3.1.0 */ - def dateadd(start: Column, days: Column): Column = Column.fn("dateadd", start, days) + def atanh(columnName: String): Column = atanh(Column(columnName)) /** - * Returns the date that is `days` days before `start` + * An expression that returns the string representation of the binary value of the given long + * column. For example, bin("12") returns "1100". * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * The number of days to subtract from `start`, can be negative to add days. A column that - * evaluates to an integer, short, or byte. - * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs + * @param e + * target column to work on. A column that evaluates to an integral. + * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ - def date_sub(start: Column, days: Int): Column = date_sub(start, lit(days)) + def bin(e: Column): Column = Column.fn("bin", e) /** - * Returns the date that is `days` days before `start` + * An expression that returns the string representation of the binary value of the given long + * column. For example, bin("12") returns "1100". * - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param days - * A column of the number of days to subtract from `start`, can be negative to add days. A - * column that evaluates to an integer, short, or byte. + * @param columnName + * target column to work on. + * @group math_funcs + * @since 1.5.0 * @return - * A date, or null if `start` was a string that could not be cast to a date. Returns a column - * that evaluates to a date. - * @group datetime_funcs - * @since 3.0.0 + * Returns a column that evaluates to a string. */ - def date_sub(start: Column, days: Column): Column = - Column.fn("date_sub", start, days) + def bin(columnName: String): Column = bin(Column(columnName)) /** - * Returns the number of days from `start` to `end`. - * - * Only considers the date part of the input. For example: - * {{{ - * datediff("2018-01-10 00:00:00", "2018-01-09 23:59:59") - * // returns 1 - * }}} + * Computes the cube-root of the given value. * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. + * @param e + * target column to compute on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def datediff(end: Column, start: Column): Column = Column.fn("datediff", end, start) + def cbrt(e: Column): Column = Column.fn("cbrt", e) /** - * Returns the number of days from `start` to `end`. - * - * Only considers the date part of the input. For example: - * {{{ - * date_diff("2018-01-10 00:00:00", "2018-01-09 23:59:59") - * // returns 1 - * }}} + * Computes the cube-root of the given column. * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. + * @param columnName + * target column to compute on. + * @group math_funcs + * @since 1.4.0 * @return - * An integer, or null if either `end` or `start` were strings that could not be cast to a - * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. - * @group datetime_funcs - * @since 3.5.0 + * Returns a column that evaluates to a double. */ - def date_diff(end: Column, start: Column): Column = Column.fn("date_diff", end, start) + def cbrt(columnName: String): Column = cbrt(Column(columnName)) /** - * Create date from the number of `days` since 1970-01-01. + * Computes the ceiling of the given value of `e` to `scale` decimal places. * - * @param days - * The number of days since 1970-01-01. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integral. Must be + * a constant. + * @group math_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a long or decimal. */ - def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) + def ceil(e: Column, scale: Column): Column = Column.fn("ceil", e, scale) /** - * Extracts the year as an integer from a given date/timestamp/string. + * Computes the ceiling of the given value of `e` to 0 decimal places. + * * @param e - * The date, timestamp or string to extract the year from. A column that evaluates to a date, - * timestamp or string. + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a long or decimal. */ - def year(e: Column): Column = Column.fn("year", e) + def ceil(e: Column): Column = Column.fn("ceil", e) /** - * Extracts the quarter as an integer from a given date/timestamp/string. - * @param e - * The date, timestamp or string to extract the quarter from. A column that evaluates to a - * date, timestamp or string. + * Computes the ceiling of the given value of `columnName` to 0 decimal places. + * + * @param columnName + * the value to compute the ceiling on. + * @group math_funcs + * @since 1.4.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a long or decimal. */ - def quarter(e: Column): Column = Column.fn("quarter", e) + def ceil(columnName: String): Column = ceil(Column(columnName)) /** - * Extracts the month as an integer from a given date/timestamp/string. + * Computes the ceiling of the given value of `e` to `scale` decimal places. + * * @param e - * The date, timestamp or string to extract the month from. A column that evaluates to a date, - * timestamp or string. + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @param scale + * parameter to control the rounding behavior. A column that evaluates to an integer. Must be + * a constant. + * @group math_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a long or decimal. */ - def month(e: Column): Column = Column.fn("month", e) + def ceiling(e: Column, scale: Column): Column = Column.fn("ceiling", e, scale) /** - * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 - * for a Sunday through to 7 for a Saturday + * Computes the ceiling of the given value of `e` to 0 decimal places. + * * @param e - * The date, timestamp or string to extract the day of the week from. A column that evaluates - * to a date, timestamp or string. + * the value to compute the ceiling on. A column that evaluates to a numeric. + * @group math_funcs + * @since 3.5.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 2.3.0 + * Returns a column that evaluates to a long or decimal. */ - def dayofweek(e: Column): Column = Column.fn("dayofweek", e) + def ceiling(e: Column): Column = Column.fn("ceiling", e) /** - * Extracts the day of the month as an integer from a given date/timestamp/string. - * @param e - * The date, timestamp or string to extract the day of the month from. A column that evaluates - * to a date, timestamp or string. - * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs + * Convert a number in a string column from one base to another. + * + * @param num + * a column to convert base for. A column that evaluates to a string. + * @param fromBase + * from base number. A column that evaluates to an integer. + * @param toBase + * to base number. A column that evaluates to an integer. + * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ - def dayofmonth(e: Column): Column = Column.fn("dayofmonth", e) + def conv(num: Column, fromBase: Int, toBase: Int): Column = + Column.fn("conv", num, lit(fromBase), lit(toBase)) /** - * Extracts the day of the month as an integer from a given date/timestamp/string. * @param e - * The date, timestamp or string to extract the day of the month from. A column that evaluates - * to a date, timestamp or string. + * angle in radians. A column that evaluates to a double. * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 3.5.0 + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def day(e: Column): Column = Column.fn("day", e) + def cos(e: Column): Column = Column.fn("cos", e) /** - * Extracts the day of the year as an integer from a given date/timestamp/string. - * @param e - * The date, timestamp or string to extract the day of the year from. A column that evaluates - * to a date, timestamp or string. + * @param columnName + * angle in radians. A column that evaluates to a double. * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * cosine of the angle, as if computed by `java.lang.Math.cos`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def dayofyear(e: Column): Column = Column.fn("dayofyear", e) + def cos(columnName: String): Column = cos(Column(columnName)) /** - * Extracts the hours as an integer from a given date/time/timestamp/string. The input may also - * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in - * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @param e - * The column to extract the hours from. A column that evaluates to a date, time, timestamp or - * string. + * hyperbolic angle. A column that evaluates to a double. * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def hour(e: Column): Column = Column.fn("hour", e) + def cosh(e: Column): Column = Column.fn("cosh", e) /** - * Extracts a part of the date/timestamp or interval source. - * - * @param field - * selects which part of the source should be extracted. - * @param source - * a date, time, timestamp or interval column from where `field` should be extracted. + * @param columnName + * hyperbolic angle * @return - * a part of the date/timestamp or interval source. Returns a column whose type depends on the - * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. - * @group datetime_funcs - * @since 3.5.0 + * hyperbolic cosine of the angle, as if computed by `java.lang.Math.cosh`. Returns a column + * that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def extract(field: Column, source: Column): Column = { - Column.fn("extract", field, source) - } + def cosh(columnName: String): Column = cosh(Column(columnName)) /** - * Extracts a part of the date/timestamp or interval source. + * @param e + * angle in radians. A column that evaluates to a double. + * @return + * cotangent of the angle. Returns a column that evaluates to a double. * - * @param field - * selects which part of the source should be extracted, and supported string values are as - * same as the fields of the equivalent function `extract`. - * @param source - * a date/timestamp or time or interval column from where `field` should be extracted. + * @group math_funcs + * @since 3.3.0 + */ + def cot(e: Column): Column = Column.fn("cot", e) + + /** + * @param e + * angle in radians. A column that evaluates to a double. * @return - * a part of the date/timestamp or interval source. Returns a column whose type depends on the - * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. - * @group datetime_funcs - * @since 3.5.0 + * cosecant of the angle. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.3.0 */ - def date_part(field: Column, source: Column): Column = { - Column.fn("date_part", field, source) - } + def csc(e: Column): Column = Column.fn("csc", e) /** - * Extracts a part of the date/timestamp or interval source. + * Returns Euler's number. * - * @param field - * selects which part of the source should be extracted, and supported string values are as - * same as the fields of the equivalent function `EXTRACT`. - * @param source - * a date/timestamp or interval column from where `field` should be extracted. - * @return - * a part of the date/timestamp or interval source. Returns a column whose type depends on the - * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. - * @group datetime_funcs + * @group math_funcs * @since 3.5.0 + * @return + * Returns a column that evaluates to a double. */ - def datepart(field: Column, source: Column): Column = { - Column.fn("datepart", field, source) - } + def e(): Column = Column.fn("e") /** - * Returns the last day of the month which the given date belongs to. For example, input - * "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015. + * Computes the exponential of the given value. * * @param e - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. + * target column to compute on. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * A date, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def last_day(e: Column): Column = Column.fn("last_day", e) + def exp(e: Column): Column = Column.fn("exp", e) /** - * Extracts the minutes as an integer from a given date/time/timestamp/string. The input may - * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in - * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. - * @param e - * The column to extract the minutes from. A column that evaluates to a date, time, timestamp - * or string. + * Computes the exponential of the given column. + * + * @param columnName + * target column to compute on. + * @group math_funcs + * @since 1.4.0 * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def minute(e: Column): Column = Column.fn("minute", e) + def exp(columnName: String): Column = exp(Column(columnName)) /** - * Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). + * Computes the exponential of the given value minus one. * * @param e - * The column to extract the day of the week from. A column that evaluates to a date, - * timestamp or string. - * @group datetime_funcs - * @since 3.5.0 + * column to calculate exponential for. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def weekday(e: Column): Column = Column.fn("weekday", e) + def expm1(e: Column): Column = Column.fn("expm1", e) /** - * @param year - * The year to build the date. A column that evaluates to an integral. - * @param month - * The month to build the date. A column that evaluates to an integral. - * @param day - * The day to build the date. A column that evaluates to an integral. + * Computes the exponential of the given column minus one. + * + * @param columnName + * column name to calculate exponential for. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * A date created from year, month and day fields. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 3.3.0 + * Returns a column that evaluates to a double. */ - def make_date(year: Column, month: Column, day: Column): Column = - Column.fn("make_date", year, month, day) + def expm1(columnName: String): Column = expm1(Column(columnName)) /** - * Returns number of months between dates `start` and `end`. - * - * A whole number is returned if both inputs have the same day of month or both are the last day - * of their respective months. Otherwise, the difference is calculated assuming 31 days per - * month. + * Computes the factorial of the given value. * - * For example: - * {{{ - * months_between("2017-11-14", "2017-07-14") // returns 4.0 - * months_between("2017-01-01", "2017-01-10") // returns 0.29032258 - * months_between("2017-06-01", "2017-06-16 12:00:00") // returns -0.5 - * }}} + * @param e + * a column to calculate factorial for. A column that evaluates to an integral. + * @group math_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a long. + */ + def factorial(e: Column): Column = Column.fn("factorial", e) + + /** + * Computes the floor of the given value of `e` to `scale` decimal places. * - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can cast to a - * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * timestamp. + * @param e + * the target column to compute the floor on. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to control the rounding behavior. A column that evaluates to + * an integral. + * @group math_funcs + * @since 3.3.0 * @return - * A double, or null if either `end` or `start` were strings that could not be cast to a - * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a - * double. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a long or decimal. */ - def months_between(end: Column, start: Column): Column = - Column.fn("months_between", end, start) + def floor(e: Column, scale: Column): Column = Column.fn("floor", e, scale) /** - * Returns number of months between dates `end` and `start`. If `roundOff` is set to true, the - * result is rounded off to 8 digits; it is not rounded otherwise. - * @param end - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param start - * A date, timestamp or string. If a string, the data must be in a format that can cast to a - * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * timestamp. - * @param roundOff - * Whether to round off the result to 8 digits. A column that evaluates to a boolean. Must be - * a constant. - * @group datetime_funcs - * @since 2.4.0 + * Computes the floor of the given value of `e` to 0 decimal places. + * + * @param e + * the target column to compute the floor on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long or decimal. */ - def months_between(end: Column, start: Column, roundOff: Boolean): Column = - Column.fn("months_between", end, start, lit(roundOff)) + def floor(e: Column): Column = Column.fn("floor", e) /** - * Returns the first date which is later than the value of the `date` column that is on the - * specified day of the week. + * Computes the floor of the given column value to 0 decimal places. * - * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first - * Sunday after 2015-07-27. - * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param dayOfWeek - * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column - * that evaluates to a string. + * @param columnName + * the target column name to compute the floor on. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a long or decimal. */ - def next_day(date: Column, dayOfWeek: String): Column = next_day(date, lit(dayOfWeek)) + def floor(columnName: String): Column = floor(Column(columnName)) /** - * Returns the first date which is later than the value of the `date` column that is on the - * specified day of the week. - * - * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first - * Sunday after 2015-07-27. + * Returns the greatest value of the list of values, skipping null values. This function takes + * at least 2 parameters. It will return null iff all parameters are null. * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param dayOfWeek - * A column of the day of week. Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", - * "Fri", "Sat", "Sun". A column that evaluates to a string. + * @param exprs + * columns to check for greatest value. A column that evaluates to any type. + * @group math_funcs + * @since 1.5.0 * @return - * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` - * was an invalid value. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 3.2.0 + * Returns a column of the same type as the input. */ - def next_day(date: Column, dayOfWeek: Column): Column = - Column.fn("next_day", date, dayOfWeek) + @scala.annotation.varargs + def greatest(exprs: Column*): Column = Column.fn("greatest", exprs: _*) /** - * Extracts the seconds as an integer from a given date/time/timestamp/string. The input may - * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in - * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. - * @param e - * The column to extract the seconds from. A column that evaluates to a date, time, timestamp - * or string. - * @return - * An integer, or null if the input was a string that could not be cast to a timestamp. - * Returns a column that evaluates to an integer. - * @group datetime_funcs + * Returns the greatest value of the list of column names, skipping null values. This function + * takes at least 2 parameters. It will return null iff all parameters are null. + * + * @param columnName + * the first column name to check for greatest value. A column of a comparable type. + * @param columnNames + * the remaining column names to check for greatest value. Columns of a comparable type. + * @group math_funcs * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ - def second(e: Column): Column = Column.fn("second", e) + @scala.annotation.varargs + def greatest(columnName: String, columnNames: String*): Column = { + greatest((columnName +: columnNames).map(Column.apply): _*) + } /** - * Extracts the week number as an integer from a given date/timestamp/string. - * - * A week is considered to start on a Monday and week 1 is the first week with more than 3 days, - * as defined by ISO 8601 + * Computes hex value of the given column. * - * @param e - * The column to extract the week number from. A column that evaluates to a date, timestamp or - * string. - * @return - * An integer, or null if the input was a string that could not be cast to a date. Returns a - * column that evaluates to an integer. - * @group datetime_funcs + * @param column + * target column to work on. A column that evaluates to an integral, string or binary. + * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a string. */ - def weekofyear(e: Column): Column = Column.fn("weekofyear", e) + def hex(column: Column): Column = Column.fn("hex", column) /** - * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string - * representing the timestamp of that moment in the current system time zone in the yyyy-MM-dd - * HH:mm:ss format. + * Inverse of hex. Interprets each pair of characters as a hexadecimal number and converts to + * the byte representation of number. * - * @param ut - * A number of a type that is castable to a long, such as string or integer. Can be negative - * for timestamps before the unix epoch - * @return - * A string, or null if the input was a string that could not be cast to a long. Returns a - * column that evaluates to a string. - * @group datetime_funcs + * @param column + * target column to work on. A column that evaluates to a string. + * @group math_funcs * @since 1.5.0 + * @return + * Returns a column that evaluates to a binary. */ - def from_unixtime(ut: Column): Column = Column.fn("from_unixtime", ut) + def unhex(column: Column): Column = Column.fn("unhex", column) /** - * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string - * representing the timestamp of that moment in the current system time zone in the given - * format. - * - * See Datetime - * Patterns for valid date and time format patterns + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param ut - * A number of a type that is castable to a long, such as string or integer. Can be negative - * for timestamps before the unix epoch - * @param f - * A date time pattern that the input will be formatted to + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * A string, or null if `ut` was a string that could not be cast to a long or `f` was an - * invalid date time pattern. Returns a column that evaluates to a string. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def from_unixtime(ut: Column, f: String): Column = - Column.fn("from_unixtime", ut, lit(f)) + def hypot(l: Column, r: Column): Column = Column.fn("hypot", l, r) /** - * Returns the current Unix timestamp (in seconds) as a long. - * - * @note - * All calls of `unix_timestamp` within the same query return the same value (i.e. the current - * timestamp is calculated at the start of query evaluation). + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @group datetime_funcs - * @since 1.5.0 + * @param l + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a double. */ - def unix_timestamp(): Column = unix_timestamp(current_timestamp()) + def hypot(l: Column, rightName: String): Column = hypot(l, Column(rightName)) /** - * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds), using the - * default timezone and the default locale. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param s - * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` - * format + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * A long, or null if the input was a string not of the correct format. Returns a column that - * evaluates to a long. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def unix_timestamp(s: Column): Column = Column.fn("unix_timestamp", s) + def hypot(leftName: String, r: Column): Column = hypot(Column(leftName), r) /** - * Converts time string with given pattern to Unix timestamp (in seconds). - * - * See Datetime - * Patterns for valid date and time format patterns + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param s - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * string, date, or timestamp. - * @param p - * A date time pattern detailing the format of `s` when `s` is a string. A column that - * evaluates to a string. + * @param leftName + * a leg. A column that evaluates to a numeric. + * @param rightName + * b leg. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid - * format. Returns a column that evaluates to a long. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def unix_timestamp(s: Column, p: String): Column = - Column.fn("unix_timestamp", s, lit(p)) + def hypot(leftName: String, rightName: String): Column = + hypot(Column(leftName), Column(rightName)) /** - * Parses a string value to a time value. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param str - * A string to be parsed to time. A column that evaluates to a string. + * @param l + * a leg. A column that evaluates to a numeric. + * @param r + * b leg. A column that evaluates to a numeric. Must be a constant. + * @group math_funcs + * @since 1.4.0 * @return - * A time, or raises an error if the input is malformed. Returns a column that evaluates to a - * time. - * - * @group datetime_funcs - * @since 4.1.0 + * Returns a column that evaluates to a double. */ - def to_time(str: Column): Column = { - Column.fn("to_time", str) - } + def hypot(l: Column, r: Double): Column = hypot(l, lit(r)) /** - * Parses a string value to a time value. - * - * See Datetime - * Patterns for valid time format patterns. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param str - * A string to be parsed to time. - * @param format - * A time format pattern to follow. A column that evaluates to a string. + * @param leftName + * The a leg of the triangle. A column that evaluates to a numeric. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @group math_funcs + * @since 1.4.0 * @return - * A time, or raises an error if the input is malformed. Returns a column that evaluates to a - * time. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column that evaluates to a double. */ - def to_time(str: Column, format: Column): Column = { - Column.fn("to_time", str, format) - } + def hypot(leftName: String, r: Double): Column = hypot(Column(leftName), r) /** - * Converts to a timestamp by casting rules to `TimestampType`. + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param s - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a string, date, timestamp, or numeric. - * @return - * A timestamp, or null if the input was a string that could not be cast to a timestamp. - * Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 2.2.0 + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param r + * The b leg of the triangle. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. */ - def to_timestamp(s: Column): Column = Column.fn("to_timestamp", s) + def hypot(l: Double, r: Column): Column = hypot(lit(l), r) /** - * Converts time string with the given pattern to timestamp. - * - * See Datetime - * Patterns for valid date and time format patterns + * Computes `sqrt(a^2^ + b^2^)` without intermediate overflow or underflow. * - * @param s - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a string, date, timestamp, or numeric. - * @param fmt - * A date time pattern detailing the format of `s` when `s` is a string. A column that - * evaluates to a string. + * @param l + * The a leg of the triangle. A column that evaluates to a numeric. Must be a constant. + * @param rightName + * The b leg of the triangle. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was - * an invalid format. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 2.2.0 + * Returns a column that evaluates to a double. */ - def to_timestamp(s: Column, fmt: String): Column = Column.fn("to_timestamp", s, lit(fmt)) + def hypot(l: Double, rightName: String): Column = hypot(l, Column(rightName)) /** - * Parses a string value to a time value. + * Returns the least value of the list of values, skipping null values. This function takes at + * least 2 parameters. It will return null iff all parameters are null. * - * @param str - * A string to be parsed to time. A column that evaluates to a string. + * @param exprs + * The values to be compared. Columns that evaluate to a comparable type. + * @group math_funcs + * @since 1.5.0 * @return - * A time, or null if the input is malformed. Returns a column that evaluates to a time. - * - * @group datetime_funcs - * @since 4.1.0 + * Returns a column of the same type as the input. */ - def try_to_time(str: Column): Column = { - Column.fn("try_to_time", str) - } + @scala.annotation.varargs + def least(exprs: Column*): Column = Column.fn("least", exprs: _*) /** - * Parses a string value to a time value. - * - * See Datetime - * Patterns for valid time format patterns. + * Returns the least value of the list of column names, skipping null values. This function + * takes at least 2 parameters. It will return null iff all parameters are null. * - * @param str - * A string to be parsed to time. - * @param format - * A time format pattern to follow. A column that evaluates to a string. + * @param columnName + * The name of the first column to be compared. A column of a comparable type. + * @param columnNames + * The names of the remaining columns to be compared. Columns of a comparable type. + * @group math_funcs + * @since 1.5.0 * @return - * A time, or null if the input is malformed. Returns a column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column of the same type as the input. */ - def try_to_time(str: Column, format: Column): Column = { - Column.fn("try_to_time", str, format) + @scala.annotation.varargs + def least(columnName: String, columnNames: String*): Column = { + least((columnName +: columnNames).map(Column.apply): _*) } /** - * Parses the `s` with the `format` to a timestamp. The function always returns null on an - * invalid input with`/`without ANSI SQL mode enabled. The result data type is consistent with - * the value of configuration `spark.sql.timestampType`. + * Computes the natural logarithm of the given value. * - * @param s - * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. - * @param format - * Format to use to convert timestamp values. A column that evaluates to a string. - * @group datetime_funcs + * @param e + * The value to compute the natural logarithm of. A column that evaluates to a numeric. + * @group math_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def try_to_timestamp(s: Column, format: Column): Column = - Column.fn("try_to_timestamp", s, format) + def ln(e: Column): Column = Column.fn("ln", e) /** - * Parses the `s` to a timestamp. The function always returns null on an invalid input - * with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The result - * data type is consistent with the value of configuration `spark.sql.timestampType`. + * Computes the natural logarithm of the given value. * - * @param s - * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * The value to compute the natural logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) + def log(e: Column): Column = ln(e) /** - * Converts the column into `DateType` by casting rules to `DateType`. + * Computes the natural logarithm of the given column. * - * @param e - * Input column of values to convert. A column that evaluates to a string, date, or timestamp. - * @group datetime_funcs - * @since 1.5.0 + * @param columnName + * The name of the column to compute the natural logarithm of. A column that evaluates to a + * numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a double. */ - def to_date(e: Column): Column = Column.fn("to_date", e) + def log(columnName: String): Column = log(Column(columnName)) /** - * Converts the column into a `DateType` with a specified format - * - * See Datetime - * Patterns for valid date and time format patterns + * Returns the first argument-base logarithm of the second argument. * - * @param e - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * string, date, or timestamp. - * @param fmt - * A date time pattern detailing the format of `e` when `e`is a string. A column that - * evaluates to a string. + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. + * @param a + * The value to compute the logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an - * invalid format. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 2.2.0 + * Returns a column that evaluates to a double. */ - def to_date(e: Column, fmt: String): Column = Column.fn("to_date", e, lit(fmt)) + def log(base: Double, a: Column): Column = Column.fn("log", lit(base), a) /** - * This is a special version of `to_date` that performs the same operation, but returns a NULL - * value instead of raising an error if date cannot be created. + * Returns the first argument-base logarithm of the second argument. * - * @param e - * Input column of values to convert. A column that evaluates to a string, date, or timestamp. - * @group datetime_funcs - * @since 4.1.0 + * @param base + * The base of the logarithm. A column that evaluates to a numeric. Must be a constant. + * @param columnName + * The name of the column to compute the logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a double. */ - def try_to_date(e: Column): Column = Column.fn("try_to_date", e) + def log(base: Double, columnName: String): Column = log(base, Column(columnName)) /** - * This is a special version of `to_date` that performs the same operation, but returns a NULL - * value instead of raising an error if date cannot be created. + * Computes the logarithm of the given value in base 10. * * @param e - * Input column of values to convert. A column that evaluates to a string, date, or timestamp. - * @param fmt - * Format to use to convert date values. A column that evaluates to a string. Must be a - * constant. - * @group datetime_funcs - * @since 4.1.0 + * The value to compute the base-10 logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a date. + * Returns a column that evaluates to a double. */ - def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) + def log10(e: Column): Column = Column.fn("log10", e) /** - * Returns the number of days since 1970-01-01. + * Computes the logarithm of the given value in base 10. * - * @param e - * Input column of values to convert. A column that evaluates to a date. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName + * The name of the column to compute the base-10 logarithm of. A column that evaluates to a + * numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a double. */ - def unix_date(e: Column): Column = Column.fn("unix_date", e) + def log10(columnName: String): Column = log10(Column(columnName)) /** - * Returns the number of microseconds since 1970-01-01 00:00:00 UTC. + * Computes the natural logarithm of the given value plus one. * * @param e - * Input column of values to convert. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * The value to compute the natural logarithm of the value plus one. A column that evaluates + * to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a double. */ - def unix_micros(e: Column): Column = Column.fn("unix_micros", e) + def log1p(e: Column): Column = Column.fn("log1p", e) /** - * Returns the number of nanoseconds since 1970-01-01 00:00:00 UTC for a nanosecond-precision - * timestamp (`TIMESTAMP_LTZ(p)` / `TIMESTAMP_NTZ(p)`, `p` in `[7, 9]`). The result is a - * lossless `DECIMAL(21, 0)`. + * Computes the natural logarithm of the given column plus one. * - * @param e - * input column of nanosecond-precision timestamp values to convert. A column that evaluates - * to a timestamp. - * @group datetime_funcs - * @since 4.3.0 + * @param columnName + * The name of the column to compute the natural logarithm of the value plus one. A column + * that evaluates to a numeric. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column that evaluates to a double. */ - def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) + def log1p(columnName: String): Column = log1p(Column(columnName)) /** - * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of - * precision. + * Computes the logarithm of the given column in base 2. * - * @param e - * input column of values to convert. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * @param expr + * The value to compute the base-2 logarithm of. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a double. */ - def unix_millis(e: Column): Column = Column.fn("unix_millis", e) + def log2(expr: Column): Column = Column.fn("log2", expr) /** - * Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of - * precision. + * Computes the logarithm of the given value in base 2. * - * @param e - * input column of values to convert. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a long. - */ - def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) + * @param columnName + * a column to calculate logarithm for. A column that evaluates to a double. + * @group math_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a double. + */ + def log2(columnName: String): Column = log2(Column(columnName)) /** - * Returns date truncated to the unit specified by the format. - * - * For example, `trunc("2018-11-19 12:01:19", "year")` returns 2018-01-01 - * - * @param date - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a - * date. - * @param format: - * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month - * Other options are: 'week', 'quarter'. A column that evaluates to a string. + * Returns the negated value. * + * @param e + * column to calculate negative value for. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 * @return - * A date, or null if `date` was a string that could not be cast to a date or `format` was an - * invalid value. Returns a column that evaluates to a date. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column of the same type as the input. */ - def trunc(date: Column, format: String): Column = Column.fn("trunc", date, lit(format)) + def negative(e: Column): Column = Column.fn("negative", e) /** - * Returns timestamp truncated to the unit specified by the format. - * - * For example, `date_trunc("year", "2018-11-19 12:01:19")` returns 2018-01-01 00:00:00 + * Returns Pi. * - * @param format: - * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', - * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', - * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. - * @param timestamp - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. + * @group math_funcs + * @since 3.5.0 * @return - * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or - * `format` was an invalid value. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 2.3.0 + * Returns a column that evaluates to a double. */ - def date_trunc(format: String, timestamp: Column): Column = - Column.fn("date_trunc", lit(format), timestamp) + def pi(): Column = Column.fn("pi") /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders - * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 - * 03:40:00.0'. + * Returns the value. * - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. It should be in - * the format of either region-based zone IDs or zone offsets. Region IDs must have the form - * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format - * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A - * column that evaluates to a string. + * @param e + * input value column. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 3.5.0 * @return - * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column of the same type as the input. */ - def from_utc_timestamp(ts: Column, tz: String): Column = from_utc_timestamp(ts, lit(tz)) + def positive(e: Column): Column = Column.fn("positive", e) /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders - * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 - * 03:40:00.0'. - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. A column that - * evaluates to a string. - * @group datetime_funcs - * @since 2.4.0 + * Returns the value of the first argument raised to the power of the second argument. + * + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def from_utc_timestamp(ts: Column, tz: Column): Column = - Column.fn("from_utc_timestamp", ts, tz) + def pow(l: Column, r: Column): Column = Column.fn("power", l, r) /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time - * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield - * '2017-07-14 01:40:00.0'. + * Returns the value of the first argument raised to the power of the second argument. * - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. It should be in - * the format of either region-based zone IDs or zone offsets. Region IDs must have the form - * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format - * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases - * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A - * column that evaluates to a string. + * @param l + * the base number. A column that evaluates to a double. + * @param rightName + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was - * an invalid value. Returns a column that evaluates to a timestamp. - * @group datetime_funcs - * @since 1.5.0 + * Returns a column that evaluates to a double. */ - def to_utc_timestamp(ts: Column, tz: String): Column = to_utc_timestamp(ts, lit(tz)) + def pow(l: Column, rightName: String): Column = pow(l, Column(rightName)) /** - * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time - * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield - * '2017-07-14 01:40:00.0'. - * @param ts - * A date, timestamp or string. If a string, the data must be in a format that can be cast to - * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to - * a timestamp. - * @param tz - * A string detailing the time zone ID that the input should be adjusted to. A column that - * evaluates to a string. - * @group datetime_funcs - * @since 2.4.0 + * Returns the value of the first argument raised to the power of the second argument. + * + * @param leftName + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) + def pow(leftName: String, r: Column): Column = pow(Column(leftName), r) /** - * Bucketize rows into one or more time windows given a timestamp specifying column. Window - * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window - * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in - * the order of months are not supported. The following example takes the average stock price - * for a one minute window every 10 seconds starting 5 seconds after the hour: - * - * {{{ - * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType - * df.groupBy(window($"timestamp", "1 minute", "10 seconds", "5 seconds"), $"stockId") - * .agg(mean("price")) - * }}} - * - * The windows will look like: - * - * {{{ - * 09:00:05-09:01:05 - * 09:00:15-09:01:15 - * 09:00:25-09:01:25 ... - * }}} - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. + * Returns the value of the first argument raised to the power of the second argument. * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param windowDuration - * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that - * the duration is a fixed length of time, and does not vary over time according to a - * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A - * column that evaluates to a string. - * @param slideDuration - * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will - * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. - * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This - * duration is likewise absolute, and does not vary according to a calendar. A column that - * evaluates to a string. - * @param startTime - * The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals. - * For example, in order to have hourly tumbling windows that start 15 minutes past the hour, - * e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`. A column that - * evaluates to a string. + * @param leftName + * the base number. + * @param rightName + * the exponent number. + * @group math_funcs + * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. + */ + def pow(leftName: String, rightName: String): Column = pow(Column(leftName), Column(rightName)) + + /** + * Returns the value of the first argument raised to the power of the second argument. * - * @group datetime_funcs - * @since 2.0.0 + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - def window( - timeColumn: Column, - windowDuration: String, - slideDuration: String, - startTime: String): Column = - Column.fn("window", timeColumn, lit(windowDuration), lit(slideDuration), lit(startTime)) + def pow(l: Column, r: Double): Column = pow(l, lit(r)) /** - * Bucketize rows into one or more time windows given a timestamp specifying column. Window - * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window - * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in - * the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 - * UTC. The following example takes the average stock price for a one minute window every 10 - * seconds: + * Returns the value of the first argument raised to the power of the second argument. * - * {{{ - * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType - * df.groupBy(window($"timestamp", "1 minute", "10 seconds"), $"stockId") - * .agg(mean("price")) - * }}} + * @param leftName + * the base number. + * @param r + * the exponent number. + * @group math_funcs + * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. + */ + def pow(leftName: String, r: Double): Column = pow(Column(leftName), r) + + /** + * Returns the value of the first argument raised to the power of the second argument. * - * The windows will look like: - * - * {{{ - * 09:00:00-09:01:00 - * 09:00:10-09:01:10 - * 09:00:20-09:01:20 ... - * }}} - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. - * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param windowDuration - * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that - * the duration is a fixed length of time, and does not vary over time according to a - * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A - * column that evaluates to a string. - * @param slideDuration - * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will - * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. - * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This - * duration is likewise absolute, and does not vary according to a calendar. A column that - * evaluates to a string. - * - * @group datetime_funcs - * @since 2.0.0 + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { - window(timeColumn, windowDuration, slideDuration, "0 second") - } + def pow(l: Double, r: Column): Column = pow(lit(l), r) /** - * Generates tumbling time windows given a timestamp specifying column. Window starts are - * inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) - * but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of - * months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC. The - * following example takes the average stock price for a one minute tumbling window: - * - * {{{ - * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType - * df.groupBy(window($"timestamp", "1 minute"), $"stockId") - * .agg(mean("price")) - * }}} - * - * The windows will look like: - * - * {{{ - * 09:00:00-09:01:00 - * 09:01:00-09:02:00 - * 09:02:00-09:03:00 ... - * }}} - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. - * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param windowDuration - * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column - * that evaluates to a string. + * Returns the value of the first argument raised to the power of the second argument. * - * @group datetime_funcs - * @since 2.0.0 + * @param l + * the base number. + * @param rightName + * the exponent number. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - def window(timeColumn: Column, windowDuration: String): Column = { - window(timeColumn, windowDuration, windowDuration, "0 second") - } + def pow(l: Double, rightName: String): Column = pow(l, Column(rightName)) /** - * Extracts the event time from the window column. - * - * The window column is of StructType { start: Timestamp, end: Timestamp } where start is - * inclusive and end is exclusive. Since event time can support microsecond precision, - * window_time(window) = window.end - 1 microsecond. - * - * @param windowColumn - * The window column (typically produced by window aggregation) of type StructType { start: - * Timestamp, end: Timestamp }. A column that evaluates to a struct. + * Returns the value of the first argument raised to the power of the second argument. * - * @group datetime_funcs - * @since 3.4.0 + * @param l + * the base number. A column that evaluates to a double. + * @param r + * the exponent number. A column that evaluates to a double. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) + def power(l: Column, r: Column): Column = Column.fn("power", l, r) /** - * Generates session window given a timestamp specifying column. - * - * Session window is one of dynamic windows, which means the length of window is varying - * according to the given inputs. The length of session window is defined as "the timestamp of - * latest input of the session + gap duration", so when the new inputs are bound to the current - * session window, the end time of session window can be expanded according to the new inputs. - * - * Windows can support microsecond precision. gapDuration in the order of months are not - * supported. - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. - * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param gapDuration - * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check - * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column - * that evaluates to a string. + * Returns the positive value of dividend mod divisor. * - * @group datetime_funcs - * @since 3.2.0 + * @param dividend + * the column that contains dividend, or the specified dividend value. A column that evaluates + * to a numeric. + * @param divisor + * the column that contains divisor, or the specified divisor value. A column that evaluates + * to a numeric. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column of the same type as the input. */ - def session_window(timeColumn: Column, gapDuration: String): Column = - session_window(timeColumn, lit(gapDuration)) + def pmod(dividend: Column, divisor: Column): Column = Column.fn("pmod", dividend, divisor) /** - * Generates session window given a timestamp specifying column. - * - * Session window is one of dynamic windows, which means the length of window is varying - * according to the given inputs. For static gap duration, the length of session window is - * defined as "the timestamp of latest input of the session + gap duration", so when the new - * inputs are bound to the current session window, the end time of session window can be - * expanded according to the new inputs. - * - * Besides a static gap duration value, users can also provide an expression to specify gap - * duration dynamically based on the input row. With dynamic gap duration, the closing of a - * session window does not depend on the latest input anymore. A session window's range is the - * union of all events' ranges which are determined by event start time and evaluated gap - * duration during the query execution. Note that the rows with negative or zero gap duration - * will be filtered out from the aggregation. - * - * Windows can support microsecond precision. gapDuration in the order of months are not - * supported. - * - * For a streaming query, you may use the function `current_timestamp` to generate windows on - * processing time. + * Returns the double value that is closest in value to the argument and is equal to a + * mathematical integer. * - * @param timeColumn - * The column or the expression to use as the timestamp for windowing by time. The time column - * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. - * @param gapDuration - * A column specifying the timeout of the session. It could be static value, e.g. `10 - * minutes`, `1 second`, or an expression/UDF that specifies gap duration dynamically based on - * the input row. A column that evaluates to a string or interval. + * @param e + * target column to compute on. A column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 + * @return + * Returns a column that evaluates to a double. + */ + def rint(e: Column): Column = Column.fn("rint", e) + + /** + * Returns the double value that is closest in value to the argument and is equal to a + * mathematical integer. * - * @group datetime_funcs - * @since 3.2.0 + * @param columnName + * the numeric column name to round to the closest integer. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a double. */ - def session_window(timeColumn: Column, gapDuration: Column): Column = - Column.fn("session_window", timeColumn, gapDuration) + def rint(columnName: String): Column = rint(Column(columnName)) /** - * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. + * Returns the value of the column `e` rounded to 0 decimal places with HALF_UP round mode. + * * @param e - * unix time values. A column that evaluates to a numeric. - * @group datetime_funcs - * @since 3.1.0 + * the value to round. A column that evaluates to a numeric. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column of the same type as the input. */ - def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) + def round(e: Column): Column = round(e, 0) /** - * Creates timestamp from the number of milliseconds since UTC epoch. + * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * * @param e - * unix time values. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column of the same type as the input. */ - def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) + def round(e: Column, scale: Int): Column = Column.fn("round", e, lit(scale)) /** - * Creates timestamp from the number of microseconds since UTC epoch. + * Round the value of `e` to `scale` decimal places with HALF_UP round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * * @param e - * unix time values. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column of the same type as the input. */ - def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) + def round(e: Column, scale: Column): Column = Column.fn("round", e, scale) /** - * Creates a timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from - * the number of nanoseconds since UTC epoch. + * Truncates the value of `e` toward zero to 0 decimal places. * * @param e - * nanosecond values since the UTC epoch. A column that evaluates to an integral or decimal. - * @group datetime_funcs - * @since 4.3.0 + * the value to truncate. A column that evaluates to a numeric. * @return - * Returns a column that evaluates to a timestamp. + * Returns a column of the same type as the input, except that a decimal input may return a + * decimal of different precision and scale. + * @group math_funcs + * @since 4.4.0 */ - def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) + def truncate(e: Column): Column = truncate(e, 0) /** - * Gets the difference between the timestamps in the specified units by truncating the fraction - * part. + * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than + * or equal to 0, or to the left of the decimal point when `scale` is less than 0. * - * @param unit - * the units of the difference between the given timestamps, e.g. 'YEAR', 'MONTH', 'DAY', - * 'HOUR'. A column that evaluates to a string. Must be a constant. - * @param start - * A timestamp which the expression subtracts from `end`. A column that evaluates to a - * timestamp. - * @param end - * A timestamp from which the expression subtracts `start`. A column that evaluates to a - * timestamp. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * the value to truncate. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to keep. A column that evaluates to an integral. Must be a + * constant. * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input, except that a decimal input may return a + * decimal of different precision and scale. + * @group math_funcs + * @since 4.4.0 */ - def timestamp_diff(unit: String, start: Column, end: Column): Column = - Column.internalFn("timestampdiff", lit(unit), start, end) + def truncate(e: Column, scale: Int): Column = Column.fn("truncate", e, lit(scale)) /** - * Adds the specified number of units to the given timestamp. + * Truncates the value of `e` toward zero to `scale` decimal places when `scale` is greater than + * or equal to 0, or to the left of the decimal point when `scale` is less than 0. * - * @param unit - * the units of datetime to add, e.g. 'YEAR', 'MONTH', 'DAY', 'HOUR'. A column that evaluates - * to a string. Must be a constant. - * @param quantity - * the number of units of time to add. A column that evaluates to an integral. - * @param ts - * A timestamp to which to add. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * the value to truncate. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to keep. A column that evaluates to an integral. Must be a + * constant. * @return - * Returns a column of the same type as the input. + * Returns a column of the same type as the input, except that a decimal input may return a + * decimal of different precision and scale. + * @group math_funcs + * @since 4.4.0 */ - def timestamp_add(unit: String, quantity: Column, ts: Column): Column = - Column.internalFn("timestampadd", lit(unit), quantity, ts) + def truncate(e: Column, scale: Column): Column = Column.fn("truncate", e, scale) /** - * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets - * aligned to the default origin (1970-01-01 00:00:00). For `TIMESTAMP_NTZ`, bucketing is - * performed in UTC. For `TIMESTAMP`, year-month interval buckets and calendar-day components of - * day-time interval buckets align to the session time zone. + * Returns the value of the column `e` rounded to 0 decimal places with HALF_EVEN round mode. * - * @param bucketSize - * A day-time or year-month interval defining the bucket size. Must be positive and foldable. - * @param ts - * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. - * @group datetime_funcs - * @since 4.2.0 + * @param e + * the value to round. A column that evaluates to a numeric. + * @group math_funcs + * @since 2.0.0 * @return * Returns a column of the same type as the input. */ - def time_bucket(bucketSize: Column, ts: Column): Column = - Column.fn("time_bucket", bucketSize, ts) + def bround(e: Column): Column = bround(e, 0) /** - * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets - * aligned to `origin`. For `TIMESTAMP_NTZ`, bucketing is performed in UTC. For `TIMESTAMP`, - * year-month interval buckets and calendar-day components of day-time interval buckets align to - * the session time zone. + * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * - * @param bucketSize - * A day-time or year-month interval defining the bucket size. Must be positive and foldable. - * @param ts - * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. - * @param origin - * Alignment anchor. Must be the same type as `ts` and must be foldable. - * @group datetime_funcs - * @since 4.2.0 + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 2.0.0 * @return * Returns a column of the same type as the input. */ - def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = - Column.fn("time_bucket", bucketSize, ts, origin) + def bround(e: Column, scale: Int): Column = Column.fn("bround", e, lit(scale)) /** - * Returns the difference between two times, measured in specified units. Throws a - * SparkIllegalArgumentException, in case the specified unit is not supported. + * Round the value of `e` to `scale` decimal places with HALF_EVEN round mode if `scale` is + * greater than or equal to 0 or at integral part when `scale` is less than 0. * - * @param unit - * A STRING representing the unit of the time difference. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A - * column that evaluates to a string. - * @param start - * A starting TIME. A column that evaluates to a time. - * @param end - * An ending TIME. A column that evaluates to a time. + * @param e + * the value to round. A column that evaluates to a numeric. + * @param scale + * the number of decimal places to round to. A column that evaluates to an integral. Must be a + * constant. + * @group math_funcs + * @since 4.0.0 * @return - * The difference between `end` and `start` times, measured in specified units. Returns a - * column that evaluates to a long. - * @note - * If any of the inputs is `NULL`, the result is `NULL`. - * @group datetime_funcs - * @since 4.1.0 + * Returns a column of the same type as the input. */ - def time_diff(unit: Column, start: Column, end: Column): Column = { - Column.fn("time_diff", unit, start, end) - } + def bround(e: Column, scale: Column): Column = Column.fn("bround", e, scale) /** - * Returns `time` truncated to the `unit`. - * - * @param unit - * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", - * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A - * column that evaluates to a string. - * @param time - * A TIME to truncate. A column that evaluates to a time. + * @param e + * angle in radians. A column that evaluates to a double. * @return - * A TIME truncated to the specified unit. Returns a column that evaluates to a time. - * @note - * If any of the inputs is `NULL`, the result is `NULL`. - * @throws IllegalArgumentException - * If the `unit` is not supported. - * @group datetime_funcs - * @since 4.1.0 + * secant of the angle. Returns a column that evaluates to a double. + * + * @group math_funcs + * @since 3.3.0 */ - def time_trunc(unit: Column, time: Column): Column = { - Column.fn("time_trunc", unit, time) - } + def sec(e: Column): Column = Column.fn("sec", e) /** - * Creates a TIME from the number of seconds since midnight. + * Shift the given value numBits left. If the given value is a long value, this function will + * return a long value else it will return an integer value. * - * @param e - * seconds since midnight (0 to 86399.999999). A column that evaluates to a numeric. - * @group datetime_funcs - * @since 4.2.0 + * @group bitwise_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a time. + * Returns a column of the same type as the input. */ - def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) + @deprecated("Use shiftleft", "3.2.0") + def shiftLeft(e: Column, numBits: Int): Column = shiftleft(e, numBits) /** - * Creates a TIME from the number of milliseconds since midnight. + * Shift the given value numBits left. If the given value is a long value, this function will + * return a long value else it will return an integer value. * * @param e - * milliseconds since midnight (0 to 86399999). A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.2.0 + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift left. A column that evaluates to an integral. Must be a + * constant. + * @group bitwise_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a time. + * Returns a column of the same type as the input. */ - def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) + def shiftleft(e: Column, numBits: Int): Column = Column.fn("shiftleft", e, lit(numBits)) /** - * Creates a TIME from the number of microseconds since midnight. + * (Signed) shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. * - * @param e - * microseconds since midnight (0 to 86399999999). A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.2.0 + * @group bitwise_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a time. + * Returns a column of the same type as the input. */ - def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) + @deprecated("Use shiftright", "3.2.0") + def shiftRight(e: Column, numBits: Int): Column = shiftright(e, numBits) /** - * Extracts the number of seconds (including fractional seconds) from a TIME value. Returns a - * DECIMAL(14,6) to preserve microsecond precision. + * (Signed) shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. * * @param e - * TIME value to convert. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.2.0 + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. + * @group bitwise_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a decimal. + * Returns a column of the same type as the input. */ - def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) + def shiftright(e: Column, numBits: Int): Column = Column.fn("shiftright", e, lit(numBits)) /** - * Extracts the number of milliseconds since midnight from a TIME value. + * Unsigned shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. * - * @param e - * the TIME value to convert. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.2.0 + * @group bitwise_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) + @deprecated("Use shiftrightunsigned", "3.2.0") + def shiftRightUnsigned(e: Column, numBits: Int): Column = shiftrightunsigned(e, numBits) /** - * Extracts the number of microseconds since midnight from a TIME value. + * Unsigned shift the given value numBits right. If the given value is a long value, it will + * return a long value else it will return an integer value. * * @param e - * the TIME value to convert. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.2.0 + * the value to shift. A column that evaluates to an integral. + * @param numBits + * the number of bits to shift right. A column that evaluates to an integral. Must be a + * constant. + * @group bitwise_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) + def shiftrightunsigned(e: Column, numBits: Int): Column = + Column.fn("shiftrightunsigned", e, lit(numBits)) /** - * Parses the `timestamp` expression with the `format` expression to a timestamp with local time - * zone. Returns null with invalid input. + * Computes the signum of the given value. * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @param format - * the format used to parse the timestamp values. A column that evaluates to a string. - * @group datetime_funcs + * @param e + * the value to compute the signum of. A column that evaluates to a numeric or interval. + * @group math_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def to_timestamp_ltz(timestamp: Column, format: Column): Column = - Column.fn("to_timestamp_ltz", timestamp, format) + def sign(e: Column): Column = Column.fn("sign", e) /** - * Parses the `timestamp` expression with the default format to a timestamp with local time - * zone. The default format follows casting rules to a timestamp. Returns null with invalid - * input. + * Computes the signum of the given value. * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * the value to compute the signum of. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def to_timestamp_ltz(timestamp: Column): Column = - Column.fn("to_timestamp_ltz", timestamp) + def signum(e: Column): Column = Column.fn("signum", e) /** - * Parses the `timestamp_str` expression with the `format` expression to a timestamp without - * time zone. Returns null with invalid input. + * Computes the signum of the given column. * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @param format - * the format used to parse the timestamp values. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName + * column to compute the signum on. A column that evaluates to a numeric or interval. + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a double. */ - def to_timestamp_ntz(timestamp: Column, format: Column): Column = - Column.fn("to_timestamp_ntz", timestamp, format) + def signum(columnName: String): Column = signum(Column(columnName)) /** - * Parses the `timestamp` expression with the default format to a timestamp without time zone. - * The default format follows casting rules to a timestamp. Returns null with invalid input. - * - * @param timestamp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a timestamp. + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def to_timestamp_ntz(timestamp: Column): Column = - Column.fn("to_timestamp_ntz", timestamp) + def sin(e: Column): Column = Column.fn("sin", e) /** - * Returns the UNIX timestamp of the given time. - * - * @param timeExp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @param format - * the format used to convert the time values. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a long. + * sine of the angle, as if computed by `java.lang.Math.sin`. Returns a column that evaluates + * to a double. + * @group math_funcs + * @since 1.4.0 */ - def to_unix_timestamp(timeExp: Column, format: Column): Column = - Column.fn("to_unix_timestamp", timeExp, format) + def sin(columnName: String): Column = sin(Column(columnName)) /** - * Returns the UNIX timestamp of the given time. - * - * @param timeExp - * the input column or strings. A column that evaluates to a date, timestamp or string. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to a long. + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def to_unix_timestamp(timeExp: Column): Column = - Column.fn("to_unix_timestamp", timeExp) + def sinh(e: Column): Column = Column.fn("sinh", e) /** - * Extracts the three-letter abbreviated month name from a given date/timestamp/string. - * - * @param timeExp - * the target date/timestamp to work on. A column that evaluates to a date, timestamp or - * string. - * @group datetime_funcs - * @since 4.0.0 + * @param columnName + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to a string. + * hyperbolic sine of the given value, as if computed by `java.lang.Math.sinh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def monthname(timeExp: Column): Column = - Column.fn("monthname", timeExp) + def sinh(columnName: String): Column = sinh(Column(columnName)) /** - * Extracts the three-letter abbreviated day name from a given date/timestamp/string. - * - * @param timeExp - * the target date/timestamp to work on. A column that evaluates to a date, timestamp or - * string. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a string. + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 1.4.0 */ - def dayname(timeExp: Column): Column = - Column.fn("dayname", timeExp) + def tan(e: Column): Column = Column.fn("tan", e) /** - * Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to - * `targetTz`. - * - * @param sourceTz - * the time zone for the input timestamp. If it is missed, the current session time zone is - * used as the source time zone. A column that evaluates to a string. - * @param targetTz - * the time zone to which the input timestamp should be converted. A column that evaluates to - * a string. - * @param sourceTs - * a timestamp without time zone. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to a timestamp. + * tangent of the given value, as if computed by `java.lang.Math.tan`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = - Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) + def tan(columnName: String): Column = tan(Column(columnName)) /** - * Converts the timestamp without time zone `sourceTs` from the current time zone to `targetTz`. - * - * @param targetTz - * the time zone to which the input timestamp should be converted. A column that evaluates to - * a string. - * @param sourceTs - * a timestamp without time zone. A column that evaluates to a timestamp. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to a timestamp. + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def convert_timezone(targetTz: Column, sourceTs: Column): Column = - Column.fn("convert_timezone", targetTz, sourceTs) + def tanh(e: Column): Column = Column.fn("tanh", e) /** - * Make DayTimeIntervalType duration from days, hours, mins and secs. - * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @param secs - * the number of seconds with the fractional part in microsecond precision. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName + * hyperbolic angle. A column that evaluates to a double. * @return - * Returns a column that evaluates to an interval. + * hyperbolic tangent of the given value, as if computed by `java.lang.Math.tanh`. Returns a + * column that evaluates to a double. + * @group math_funcs + * @since 1.4.0 */ - def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = - Column.fn("make_dt_interval", days, hours, mins, secs) + def tanh(columnName: String): Column = tanh(Column(columnName)) /** - * Make DayTimeIntervalType duration from days, hours and mins. - * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a double. */ - def make_dt_interval(days: Column, hours: Column, mins: Column): Column = - Column.fn("make_dt_interval", days, hours, mins) + @deprecated("Use degrees", "2.1.0") + def toDegrees(e: Column): Column = degrees(e) /** - * Make DayTimeIntervalType duration from days and hours. - * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a double. */ - def make_dt_interval(days: Column, hours: Column): Column = - Column.fn("make_dt_interval", days, hours) + @deprecated("Use degrees", "2.1.0") + def toDegrees(columnName: String): Column = degrees(Column(columnName)) /** - * Make DayTimeIntervalType duration from days. + * Converts an angle measured in radians to an approximately equivalent angle measured in + * degrees. * - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to an interval. + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 2.1.0 */ - def make_dt_interval(days: Column): Column = - Column.fn("make_dt_interval", days) + def degrees(e: Column): Column = Column.fn("degrees", e) /** - * Make DayTimeIntervalType duration. + * Converts an angle measured in radians to an approximately equivalent angle measured in + * degrees. * - * @group datetime_funcs - * @since 3.5.0 + * @param columnName + * angle in radians. A column that evaluates to a double. * @return - * Returns a column that evaluates to an interval. + * angle in degrees, as if computed by `java.lang.Math.toDegrees`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 2.1.0 */ - def make_dt_interval(): Column = - Column.fn("make_dt_interval") + def degrees(columnName: String): Column = degrees(Column(columnName)) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. - * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @param secs - * the number of seconds with the fractional part in microsecond precision. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a double. */ - def try_make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days, hours, mins, secs) + @deprecated("Use radians", "2.1.0") + def toRadians(e: Column): Column = radians(e) /** - * Make interval from years, months, weeks, days, hours, mins and secs. - * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @param secs - * the number of seconds with the fractional part in microsecond precision. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @group math_funcs + * @since 1.4.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a double. */ - def make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_interval", years, months, weeks, days, hours, mins, secs) + @deprecated("Use radians", "2.1.0") + def toRadians(columnName: String): Column = radians(Column(columnName)) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Converts an angle measured in degrees to an approximately equivalent angle measured in + * radians. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * angle in degrees. A column that evaluates to a double. * @return - * Returns a column that evaluates to an interval. + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. + * + * @group math_funcs + * @since 2.1.0 */ - def try_make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days, hours, mins) + def radians(e: Column): Column = Column.fn("radians", e) /** - * Make interval from years, months, weeks, days, hours and mins. + * Converts an angle measured in degrees to an approximately equivalent angle measured in + * radians. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @param mins - * the number of minutes, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param columnName + * angle in degrees. A column that evaluates to a double. * @return - * Returns a column that evaluates to an interval. + * angle in radians, as if computed by `java.lang.Math.toRadians`. Returns a column that + * evaluates to a double. + * @group math_funcs + * @since 2.1.0 */ - def make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column, - mins: Column): Column = - Column.fn("make_interval", years, months, weeks, days, hours, mins) + def radians(columnName: String): Column = radians(Column(columnName)) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Returns the bucket number into which the value of this expression would fall after being + * evaluated. Note that input arguments must follow conditions listed below; otherwise, the + * method will return null. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param v + * value to compute a bucket number in the histogram. A column that evaluates to a double or + * interval. + * @param min + * minimum value of the histogram. A column that evaluates to a double or interval. + * @param max + * maximum value of the histogram. A column that evaluates to a double or interval. + * @param numBucket + * the number of buckets. A column that evaluates to a long. * @return - * Returns a column that evaluates to an interval. + * the bucket number into which the value would fall after being evaluated. Returns a column + * that evaluates to a long. + * @group math_funcs + * @since 3.5.0 */ - def try_make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days, hours) + def width_bucket(v: Column, min: Column, max: Column, numBucket: Column): Column = + Column.fn("width_bucket", v, min, max, numBucket) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Misc functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Make interval from years, months, weeks, days and hours. + * Returns the current catalog. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @param hours - * the number of hours, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def make_interval( - years: Column, - months: Column, - weeks: Column, - days: Column, - hours: Column): Column = - Column.fn("make_interval", years, months, weeks, days, hours) + def current_catalog(): Column = Column.fn("current_catalog") /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Returns the current database. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = - Column.fn("try_make_interval", years, months, weeks, days) + def current_database(): Column = Column.fn("current_database") /** - * Make interval from years, months, weeks and days. + * Returns the current schema. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @param days - * the number of days, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = - Column.fn("make_interval", years, months, weeks, days) + def current_schema(): Column = Column.fn("current_schema") /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Returns the current SQL path as a comma-separated list of qualified schema names. * - * @param years - * the number of years, positive or negative. A column that evaluates to an integral. - * @param months - * the number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * the number of weeks, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @group misc_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def try_make_interval(years: Column, months: Column, weeks: Column): Column = - Column.fn("try_make_interval", years, months, weeks) + def current_path(): Column = Column.fn("current_path") /** - * Make interval from years, months and weeks. + * Returns the user name of current execution context. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @param weeks - * The number of weeks, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def make_interval(years: Column, months: Column, weeks: Column): Column = - Column.fn("make_interval", years, months, weeks) + def current_user(): Column = Column.fn("current_user") /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Calculates the MD5 digest of a binary column and returns the value as a 32 character hex + * string. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * target column to compute on. A column that evaluates to a binary. + * @group hash_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def try_make_interval(years: Column, months: Column): Column = - Column.fn("try_make_interval", years, months) + def md5(e: Column): Column = Column.fn("md5", e) /** - * Make interval from years and months. + * Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex + * string. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * target column to compute on. A column that evaluates to a binary. + * @group hash_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def make_interval(years: Column, months: Column): Column = - Column.fn("make_interval", years, months) + def sha1(e: Column): Column = Column.fn("sha1", e) /** - * This is a special version of `make_interval` that performs the same operation, but returns a - * NULL value instead of raising an error if interval cannot be created. + * Calculates the SHA-2 family of hash functions of a binary column and returns the value as a + * hex string. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 4.0.0 + * @param e + * column to compute SHA-2 on. A column that evaluates to a binary. + * @param numBits + * one of 224, 256, 384, or 512. A column that evaluates to an integer. + * + * @group hash_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a string. */ - def try_make_interval(years: Column): Column = - Column.fn("try_make_interval", years) + def sha2(e: Column, numBits: Int): Column = { + require( + Seq(0, 224, 256, 384, 512).contains(numBits), + s"numBits $numBits is not in the permitted values (0, 224, 256, 384, 512)") + Column.fn("sha2", e, lit(numBits)) + } /** - * Make interval from years. + * Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value + * as a bigint. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs - * @since 3.5.0 + * @param e + * target column to compute on. A column that evaluates to a binary. + * @group hash_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a long. */ - def make_interval(years: Column): Column = - Column.fn("make_interval", years) + def crc32(e: Column): Column = Column.fn("crc32", e) /** - * Make interval. + * Calculates the hash code of given columns, and returns the result as an int column. * - * @group datetime_funcs - * @since 3.5.0 + * @param cols + * one or more columns to compute on. A column of any type. + * @group hash_funcs + * @since 2.0.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to an integer. */ - def make_interval(): Column = - Column.fn("make_interval") + @scala.annotation.varargs + def hash(cols: Column*): Column = Column.fn("hash", cols: _*) /** - * Create timestamp from years, months, days, hours, mins, secs and timezone fields. The result - * data type is consistent with the value of configuration `spark.sql.timestampType`. If the - * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. - * Otherwise, it will throw an error instead. + * Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, + * and returns the result as a long column. The hash computation uses an initial seed of 42. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 3.5.0 + * @param cols + * one or more columns to compute on. A column of any type. + * @group hash_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a long. */ - def make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("make_timestamp", years, months, days, hours, mins, secs, timezone) + @scala.annotation.varargs + def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) /** - * Create timestamp from years, months, days, hours, mins and secs fields. The result data type - * is consistent with the value of configuration `spark.sql.timestampType`. If the configuration - * `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it - * will throw an error instead. + * Returns a 64-bit hash value of the argument using the XXH3 algorithm. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param col + * the column to hash, which must have string or binary type. + * @group hash_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a long. */ - def make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_timestamp", years, months, days, hours, mins, secs) + def xxh3_64(col: Column): Column = Column.fn("xxh3_64", col) /** - * Create a local date-time from date, time, and timezone fields. + * Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.1.0 + * @param col + * the column to hash, which must have string or binary type. + * @group hash_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def make_timestamp(date: Column, time: Column, timezone: Column): Column = - Column.fn("make_timestamp", date, time, timezone) + def xxh3_128(col: Column): Column = Column.fn("xxh3_128", col) /** - * Create a local date-time from date and time fields. + * Returns null if the condition is true, and throws an exception otherwise. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * @param c + * The condition to check. A column that evaluates to a boolean. + * @group misc_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that always evaluates to NULL. */ - def make_timestamp(date: Column, time: Column): Column = - Column.fn("make_timestamp", date, time) + def assert_true(c: Column): Column = Column.fn("assert_true", c) /** - * Try to create a timestamp from years, months, days, hours, mins, secs and timezone fields. - * The result data type is consistent with the value of configuration `spark.sql.timestampType`. - * The function returns NULL on invalid inputs. + * Returns null if the condition is true; throws an exception with the error message otherwise. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.0.0 + * @param c + * The condition to check. A column that evaluates to a boolean. + * @param e + * The error message to throw. A column that evaluates to a string. + * @group misc_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that always evaluates to NULL. */ - def try_make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("try_make_timestamp", years, months, days, hours, mins, secs, timezone) + def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) /** - * Try to create a timestamp from years, months, days, hours, mins, and secs fields. The result - * data type is consistent with the value of configuration `spark.sql.timestampType`. The - * function returns NULL on invalid inputs. + * Throws an exception with the provided error message. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 + * @param c + * The error message to throw. A column that evaluates to a string. + * @group misc_funcs + * @since 3.1.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that always evaluates to NULL. */ - def try_make_timestamp( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_timestamp", years, months, days, hours, mins, secs) + def raise_error(c: Column): Column = Column.fn("raise_error", c) /** - * Try to create a local date-time from date, time, and timezone fields. + * Returns the user name of current execution context. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.1.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = - Column.fn("try_make_timestamp", date, time, timezone) + def user(): Column = Column.fn("user") /** - * Try to create a local date-time from date and time fields. + * Returns the user name of current execution context. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * @group misc_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def try_make_timestamp(date: Column, time: Column): Column = - Column.fn("try_make_timestamp", date, time) + def session_user(): Column = Column.fn("session_user") /** - * Create the current timestamp with local time zone from years, months, days, hours, mins, secs - * and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function - * returns NULL on invalid inputs. Otherwise, it will throw an error instead. + * Returns an universally unique identifier (UUID) string. The value is returned as a canonical + * UUID 36-character string. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) + def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) /** - * Create the current timestamp with local time zone from years, months, days, hours, mins and - * secs fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns - * NULL on invalid inputs. Otherwise, it will throw an error instead. + * Returns an universally unique identifier (UUID) string. The value is returned as a canonical + * UUID 36-character string. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 + * @param seed + * The random number seed to use. A column that evaluates to an integral. Must be a constant. + * @group misc_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a string. */ - def make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs) + def uuid(seed: Column): Column = Column.fn("uuid", seed) /** - * Try to create the current timestamp with local time zone from years, months, days, hours, - * mins, secs and timezone fields. The function returns NULL on invalid inputs. + * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the + * given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with `hex` or + * `base64` for a textual value. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @param timezone - * The time zone identifier. A column that evaluates to a string. - * @group datetime_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a timestamp. - */ - def try_make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column, - timezone: Column): Column = - Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) - - /** - * Try to create the current timestamp with local time zone from years, months, days, hours, - * mins and secs fields. The function returns NULL on invalid inputs. + * @param key + * The secret key, as a binary value. + * @param message + * The message to authenticate, as a binary value. + * @param algorithm + * The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a timestamp. + * @group misc_funcs + * @since 4.3.0 */ - def try_make_timestamp_ltz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs) + def hmac(key: Column, message: Column, algorithm: Column): Column = + Column.fn("hmac", key, message, algorithm) /** - * Create local date-time from years, months, days, hours, mins, secs fields. If the - * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. - * Otherwise, it will throw an error instead. + * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and + * SHA-256. The result is returned as raw MAC bytes; wrap it with `hex` or `base64` for a + * textual value. To use a different algorithm, call the three-argument overload. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a timestamp. + * @param key + * The secret key, as a binary value. + * @param message + * The message to authenticate, as a binary value. + * + * @group misc_funcs + * @since 4.3.0 */ - def make_timestamp_ntz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("make_timestamp_ntz", years, months, days, hours, mins, secs) + def hmac(key: Column, message: Column): Column = + Column.fn("hmac", key, message) /** - * Create a local date-time from date and time fields. - * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 - * @return - * Returns a column that evaluates to a timestamp. - */ - def make_timestamp_ntz(date: Column, time: Column): Column = - Column.fn("make_timestamp_ntz", date, time) - - /** - * Try to create a local date-time from years, months, days, hours, mins, secs fields. The - * function returns NULL on invalid inputs. + * Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. + * Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, + * `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization + * vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 + * bytes for GCM. If not provided, a random vector will be generated and prepended to the + * output. Optional additional authenticated data (AAD) is only supported for GCM. If provided + * for encryption, the identical AAD value must be provided for decryption. The default mode is + * GCM. * - * @param years - * The year to represent, from 1 to 9999. A column that evaluates to an integral. - * @param months - * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates - * to an integral. - * @param days - * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. - * @param hours - * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. - * @param mins - * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. - * @param secs - * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that - * evaluates to a numeric. - * @group datetime_funcs - * @since 4.0.0 + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param iv + * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp_ntz( - years: Column, - months: Column, - days: Column, - hours: Column, - mins: Column, - secs: Column): Column = - Column.fn("try_make_timestamp_ntz", years, months, days, hours, mins, secs) + def aes_encrypt( + input: Column, + key: Column, + mode: Column, + padding: Column, + iv: Column, + aad: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv, aad) /** - * Try to create a local date-time from date and time fields. + * Returns an encrypted value of `input`. * - * @param date - * The date to represent, in valid DATE format. A column that evaluates to a date. - * @param time - * The time to represent, in valid TIME format. A column that evaluates to a time. - * @group datetime_funcs - * @since 4.1.0 + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param iv + * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or + * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a + * binary. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a timestamp. + * Returns a column that evaluates to a binary. */ - def try_make_timestamp_ntz(date: Column, time: Column): Column = - Column.fn("try_make_timestamp_ntz", date, time) + def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = + Column.fn("aes_encrypt", input, key, mode, padding, iv) /** - * Make year-month interval from years, months. + * Returns an encrypted value of `input`. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @param months - * The number of months, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_ym_interval(years: Column, months: Column): Column = - Column.fn("make_ym_interval", years, months) + def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = + Column.fn("aes_encrypt", input, key, mode, padding) /** - * Make year-month interval from years. + * Returns an encrypted value of `input`. * - * @param years - * The number of years, positive or negative. A column that evaluates to an integral. - * @group datetime_funcs + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) + def aes_encrypt(input: Column, key: Column, mode: Column): Column = + Column.fn("aes_encrypt", input, key, mode) /** - * Make year-month interval. + * Returns an encrypted value of `input`. * - * @group datetime_funcs + * @param input + * The binary value to encrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to encrypt the data. A column that evaluates to a binary. + * @see + * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, + * Column)` + * + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to an interval. + * Returns a column that evaluates to a binary. */ - def make_ym_interval(): Column = Column.fn("make_ym_interval") - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Hash Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def aes_encrypt(input: Column, key: Column): Column = + Column.fn("aes_encrypt", input, key) /** - * Calculates the MD5 digest of a binary column and returns the value as a 32 character hex - * string. + * Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, + * 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', + * 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is + * only supported for GCM. If provided for encryption, the identical AAD value must be provided + * for decryption. The default mode is GCM. * - * @param e - * target column to compute on. A column that evaluates to a binary. - * @group hash_funcs - * @since 1.5.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def md5(e: Column): Column = Column.fn("md5", e) + def aes_decrypt( + input: Column, + key: Column, + mode: Column, + padding: Column, + aad: Column): Column = + Column.fn("aes_decrypt", input, key, mode, padding, aad) /** - * Calculates the SHA-1 digest of a binary column and returns the value as a 40 character hex - * string. + * Returns a decrypted value of `input`. * - * @param e - * target column to compute on. A column that evaluates to a binary. - * @group hash_funcs - * @since 1.5.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def sha1(e: Column): Column = Column.fn("sha1", e) + def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = + Column.fn("aes_decrypt", input, key, mode, padding) /** - * Calculates the SHA-2 family of hash functions of a binary column and returns the value as a - * hex string. + * Returns a decrypted value of `input`. * - * @param e - * column to compute SHA-2 on. A column that evaluates to a binary. - * @param numBits - * one of 224, 256, 384, or 512. A column that evaluates to an integer. + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` * - * @group hash_funcs - * @since 1.5.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def sha2(e: Column, numBits: Int): Column = { - require( - Seq(0, 224, 256, 384, 512).contains(numBits), - s"numBits $numBits is not in the permitted values (0, 224, 256, 384, 512)") - Column.fn("sha2", e, lit(numBits)) - } + def aes_decrypt(input: Column, key: Column, mode: Column): Column = + Column.fn("aes_decrypt", input, key, mode) /** - * Calculates the cyclic redundancy check value (CRC32) of a binary column and returns the value - * as a bigint. + * Returns a decrypted value of `input`. * - * @param e - * target column to compute on. A column that evaluates to a binary. - * @group hash_funcs - * @since 1.5.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @see + * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def crc32(e: Column): Column = Column.fn("crc32", e) + def aes_decrypt(input: Column, key: Column): Column = + Column.fn("aes_decrypt", input, key) /** - * Calculates the hash code of given columns, and returns the result as an int column. + * This is a special version of `aes_decrypt` that performs the same operation, but returns a + * NULL value instead of raising an error if the decryption cannot be performed. * - * @param cols - * one or more columns to compute on. A column of any type. - * @group hash_funcs - * @since 2.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @param aad + * Optional additional authenticated data. Only supported for GCM mode. This can be any + * free-form input and must be provided for both encryption and decryption. A column that + * evaluates to a binary. + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - @scala.annotation.varargs - def hash(cols: Column*): Column = Column.fn("hash", cols: _*) + def try_aes_decrypt( + input: Column, + key: Column, + mode: Column, + padding: Column, + aad: Column): Column = + Column.fn("try_aes_decrypt", input, key, mode, padding, aad) /** - * Calculates the hash code of given columns using the 64-bit variant of the xxHash algorithm, - * and returns the result as a long column. The hash computation uses an initial seed of 42. + * Returns a decrypted value of `input`. * - * @param cols - * one or more columns to compute on. A column of any type. - * @group hash_funcs - * @since 3.0.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @param padding + * Specifies how to pad messages whose length is not a multiple of the block size. Valid + * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS + * for CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - @scala.annotation.varargs - def xxhash64(cols: Column*): Column = Column.fn("xxhash64", cols: _*) + def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = + Column.fn("try_aes_decrypt", input, key, mode, padding) /** - * Returns a 64-bit hash value of the argument using the XXH3 algorithm. + * Returns a decrypted value of `input`. * - * @param col - * the column to hash, which must have string or binary type. - * @group hash_funcs - * @since 4.4.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @param mode + * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, + * GCM, CBC. A column that evaluates to a string. + * @see + * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def xxh3_64(col: Column): Column = Column.fn("xxh3_64", col) + def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = + Column.fn("try_aes_decrypt", input, key, mode) /** - * Returns a 128-bit XXH3 hash of the argument as a 32-character hex string. + * Returns a decrypted value of `input`. * - * @param col - * the column to hash, which must have string or binary type. - * @group hash_funcs - * @since 4.4.0 + * @param input + * The binary value to decrypt. A column that evaluates to a binary. + * @param key + * The passphrase to use to decrypt the data. A column that evaluates to a binary. + * @see + * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def xxh3_128(col: Column): Column = Column.fn("xxh3_128", col) + def try_aes_decrypt(input: Column, key: Column): Column = + Column.fn("try_aes_decrypt", input, key) /** * Returns a sha1 hash value as a hex string of the `col`. @@ -7303,9916 +7132,10012 @@ object functions { */ def sha(col: Column): Column = Column.fn("sha", col) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Collection Functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** - * Concatenates multiple input columns together into a single column. The function works with - * strings, binary and compatible array columns. - * - * @param exprs - * Input columns to concatenate. A column that evaluates to a string, binary or an array. - * @note - * Returns null if any of the input columns are null. + * Returns the length of the block being read, or -1 if not available. * - * @group collection_funcs - * @since 1.5.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - @scala.annotation.varargs - def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) + def input_file_block_length(): Column = Column.fn("input_file_block_length") /** - * Returns element of array at given index in value if column is array. Returns value for the - * given key in value if column is map. + * Returns the start offset of the block being read, or -1 if not available. * - * @param column - * The array or map to extract from. A column that evaluates to an array or a map. - * @param value - * The 1-based index for arrays, or the key for maps. A column. - * @group collection_funcs - * @since 2.4.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column of the element type of the input array, or the value type of the input - * map. + * Returns a column that evaluates to a long. */ - def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) + def input_file_block_start(): Column = Column.fn("input_file_block_start") /** - * (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will - * throw an error. If index < 0, accesses elements from the last to the first. The function - * always returns NULL if the index exceeds the length of the array. - * - * (map, key) - Returns value for given key. The function always returns NULL if the key is not - * contained in the map. + * Calls a method with reflection. * - * @param column - * The array or map to extract from. A column that evaluates to an array or a map. - * @param value - * The 1-based index for arrays, or the key for maps. A column. - * @group collection_funcs + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column of the element type of the input array, or the value type of the input - * map. + * Returns a column that evaluates to a string. */ - def try_element_at(column: Column, value: Column): Column = - Column.fn("try_element_at", column, value) + @scala.annotation.varargs + def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) /** - * Sorts the input array in ascending order. Null elements will be placed at the end of the - * returned array. NaN is greater than any non-NaN elements for double/float type. - * - * The elements of the input array must be orderable. For example, when the array elements are - * structs, the default comparator compares the struct fields in schema order. Therefore, all - * fields in the struct must be orderable. If the default comparator does not support the input - * type, you can specify a custom comparator. + * Calls a method with reflection. * - * @param e - * The array to sort. A column that evaluates to an array. - * @group collection_funcs - * @since 2.4.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def array_sort(e: Column): Column = Column.fn("array_sort", e) + @scala.annotation.varargs + def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) /** - * Sorts the input array based on the given comparator function. The comparator will take two - * arguments representing two elements of the array. It returns a negative integer, 0, or a - * positive integer as the first element is less than, equal to, or greater than the second - * element. If the comparator function returns null, the function will fail and raise an error. + * This is a special version of `reflect` that performs the same operation, but returns a NULL + * value instead of raising an error if the invoke method thrown exception. * - * @param e - * The array to sort. A column that evaluates to an array. - * @param comparator - * A binary comparator function that returns a negative integer, 0, or a positive integer as - * the first element is less than, equal to, or greater than the second element. - * @group collection_funcs - * @since 3.4.0 + * @group misc_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def array_sort(e: Column, comparator: (Column, Column) => Column): Column = - Column.fn("array_sort", e, createLambda(comparator)) - - private def createLambda(f: Column => Column) = { - val x = internal.UnresolvedNamedLambdaVariable("x") - val function = f(Column(x)).node - Column(internal.LambdaFunction(function, Seq(x))) - } - - private def createLambda(f: (Column, Column) => Column) = { - val x = internal.UnresolvedNamedLambdaVariable("x") - val y = internal.UnresolvedNamedLambdaVariable("y") - val function = f(Column(x), Column(y)).node - Column(internal.LambdaFunction(function, Seq(x, y))) - } - - private def createLambda(f: (Column, Column, Column) => Column) = { - val x = internal.UnresolvedNamedLambdaVariable("x") - val y = internal.UnresolvedNamedLambdaVariable("y") - val z = internal.UnresolvedNamedLambdaVariable("z") - val function = f(Column(x), Column(y), Column(z)).node - Column(internal.LambdaFunction(function, Seq(x, y, z))) - } + @scala.annotation.varargs + def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) /** - * Returns an array of elements after applying a transformation to each element in the input - * array. - * {{{ - * df.select(transform(col("i"), x => x + 1)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => transformed_col, the lambda function to transform the input column. + * Returns the Spark version. The string contains 2 fields, the first being a release version + * and the second being a git revision. * - * @group collection_funcs - * @since 3.0.0 + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def transform(column: Column, f: Column => Column): Column = - Column.fn("transform", column, createLambda(f)) + def version(): Column = Column.fn("version") /** - * Returns an array of elements after applying a transformation to each element in the input - * array. - * {{{ - * df.select(transform(col("i"), (x, i) => x + i)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * (col, index) => transformed_col, the lambda function to transform the input column given - * the index. Indices start at 0. + * Return DDL-formatted type string for the data type of the input. * - * @group collection_funcs - * @since 3.0.0 + * @param col + * The value whose data type is returned. A column of any type. + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def transform(column: Column, f: (Column, Column) => Column): Column = - Column.fn("transform", column, createLambda(f)) + def typeof(col: Column): Column = Column.fn("typeof", col) /** - * Returns whether a predicate holds for one or more elements in the array. - * {{{ - * df.select(exists(col("i"), _ % 2 === 0)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => predicate, the Boolean predicate to check the input column. + * Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default + * unless specified otherwise. * - * @group collection_funcs - * @since 3.0.0 + * @param cols + * The first column must be a constant integer for the number of rows, and the remaining + * columns are the input elements to be separated into rows. + * @group generator_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column of the same type as the input. */ - def exists(column: Column, f: Column => Column): Column = - Column.fn("exists", column, createLambda(f)) + @scala.annotation.varargs + def stack(cols: Column*): Column = Column.fn("stack", cols: _*) /** - * Returns whether a predicate holds for every element in the array. - * {{{ - * df.select(forall(col("i"), x => x % 2 === 0)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => predicate, the Boolean predicate to check the input column. + * Returns a random value with independent and identically distributed (i.i.d.) values with the + * specified range of numbers. The provided numbers specifying the minimum and maximum values of + * the range must be constant. If both of these numbers are integers, then the result will also + * be an integer. Otherwise if one or both of these are floating-point numbers, then the result + * will also be a floating-point number. * - * @group collection_funcs - * @since 3.0.0 + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column of the same type as the input. */ - def forall(column: Column, f: Column => Column): Column = - Column.fn("forall", column, createLambda(f)) + def uniform(min: Column, max: Column): Column = + uniform(min, max, lit(SparkClassUtils.random.nextLong)) /** - * Returns an array of elements for which a predicate holds in a given array. - * {{{ - * df.select(filter(col("s"), x => x % 2 === 0)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * col => predicate, the Boolean predicate to filter the input column. + * Returns a random value with independent and identically distributed (i.i.d.) values with the + * specified range of numbers, with the chosen random seed. The provided numbers specifying the + * minimum and maximum values of the range must be constant. If both of these numbers are + * integers, then the result will also be an integer. Otherwise if one or both of these are + * floating-point numbers, then the result will also be a floating-point number. * - * @group collection_funcs - * @since 3.0.0 + * @param min + * Minimum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param max + * Maximum value in the range. A column that evaluates to a numeric. Must be a constant. + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. + * @group math_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def filter(column: Column, f: Column => Column): Column = - Column.fn("filter", column, createLambda(f)) + def uniform(min: Column, max: Column, seed: Column): Column = + Column.fn("uniform", min, max, seed) /** - * Returns an array of elements for which a predicate holds in a given array. - * {{{ - * df.select(filter(col("s"), (x, i) => i % 2 === 0)) - * }}} - * - * @param column - * the input array column. A column that evaluates to an array. - * @param f - * (col, index) => predicate, the Boolean predicate to filter the input column given the - * index. Indices start at 0. + * Returns a random value with independent and identically distributed (i.i.d.) uniformly + * distributed values in [0, 1). * - * @group collection_funcs - * @since 3.0.0 + * @param seed + * Random number seed to use. A column that evaluates to an integral. Must be a constant. + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a double. */ - def filter(column: Column, f: (Column, Column) => Column): Column = - Column.fn("filter", column, createLambda(f)) + def random(seed: Column): Column = Column.fn("random", seed) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. The final state is converted into the final result by applying a finish - * function. - * {{{ - * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) - * }}} - * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value. - * @param finish - * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result. + * Returns a random value with independent and identically distributed (i.i.d.) uniformly + * distributed values in [0, 1). * - * @group collection_funcs - * @since 3.0.0 + * @group math_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column that evaluates to a double. */ - def aggregate( - expr: Column, - initialValue: Column, - merge: (Column, Column) => Column, - finish: Column => Column): Column = - Column.fn("aggregate", expr, initialValue, createLambda(merge), createLambda(finish)) + def random(): Column = random(lit(SparkClassUtils.random.nextLong)) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. - * {{{ - * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x)) - * }}} + * Returns the bit position for the given input column. * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value - * @group collection_funcs - * @since 3.0.0 + * @param col + * The input column. A column that evaluates to an integral. + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column that evaluates to a long. */ - def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = - aggregate(expr, initialValue, merge, c => c) + def bitmap_bit_position(col: Column): Column = + Column.fn("bitmap_bit_position", col) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. The final state is converted into the final result by applying a finish - * function. - * {{{ - * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) - * }}} - * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value. - * @param finish - * combined_value => final_value, the lambda function to convert the combined value of all - * inputs to final result. + * Returns the bucket number for the given input column. * - * @group collection_funcs + * @param col + * The input column. A column that evaluates to an integral. + * @group misc_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column that evaluates to a long. */ - def reduce( - expr: Column, - initialValue: Column, - merge: (Column, Column) => Column, - finish: Column => Column): Column = - Column.fn("reduce", expr, initialValue, createLambda(merge), createLambda(finish)) + def bitmap_bucket_number(col: Column): Column = + Column.fn("bitmap_bucket_number", col) /** - * Applies a binary operator to an initial state and all elements in the array, and reduces this - * to a single state. - * {{{ - * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x)) - * }}} + * Returns a bitmap with the positions of the bits set from all the values from the input + * column. The input column will most likely be bitmap_bit_position(). * - * @param expr - * the input array column. A column that evaluates to an array. - * @param initialValue - * the initial value. A column of any type. - * @param merge - * (combined_value, input_value) => combined_value, the merge function to merge an input value - * to the combined_value - * @group collection_funcs + * @param col + * The input column will most likely be bitmap_bit_position(). A column that evaluates to an + * integral. + * @group agg_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the initial value. + * Returns a column that evaluates to a binary. */ - def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = - reduce(expr, initialValue, merge, c => c) + def bitmap_construct_agg(col: Column): Column = + Column.fn("bitmap_construct_agg", col) /** - * Merge two given arrays, element-wise, into a single array using a function. If one array is - * shorter, nulls are appended at the end to match the length of the longer array, before - * applying the function. - * {{{ - * df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y)) - * }}} - * - * @param left - * the left input array column. A column that evaluates to an array. - * @param right - * the right input array column. A column that evaluates to an array. - * @param f - * (lCol, rCol) => col, the lambda function to merge two input columns into one column. + * Returns the number of set bits in the input bitmap. * - * @group collection_funcs - * @since 3.0.0 + * @param col + * The input bitmap. A column that evaluates to a binary. + * @group misc_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a long. */ - def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = - Column.fn("zip_with", left, right, createLambda(f)) + def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) /** - * Applies a function to every key-value pair in a map and returns a map with the results of - * those applications as the new keys for the pairs. - * {{{ - * df.select(transform_keys(col("i"), (k, v) => k + v)) - * }}} - * - * @param expr - * the input map column. A column that evaluates to a map. - * @param f - * (key, value) => new_key, the lambda function to transform the key of input map column + * Returns a bitmap that is the bitwise AND of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * - * @group collection_funcs - * @since 3.0.0 + * @param left + * A column that evaluates to a binary bitmap. + * @param right + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a binary bitmap. */ - def transform_keys(expr: Column, f: (Column, Column) => Column): Column = - Column.fn("transform_keys", expr, createLambda(f)) + def bitmap_and(left: Column, right: Column): Column = Column.fn("bitmap_and", left, right) /** - * Applies a function to every key-value pair in a map and returns a map with the results of - * those applications as the new values for the pairs. - * {{{ - * df.select(transform_values(col("i"), (k, v) => k + v)) - * }}} - * - * @param expr - * the input map column. A column that evaluates to a map. - * @param f - * (key, value) => new_value, the lambda function to transform the value of input map column + * Returns a bitmap that is the bitwise OR of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * - * @group collection_funcs - * @since 3.0.0 + * @param left + * A column that evaluates to a binary bitmap. + * @param right + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a binary bitmap. */ - def transform_values(expr: Column, f: (Column, Column) => Column): Column = - Column.fn("transform_values", expr, createLambda(f)) + def bitmap_or(left: Column, right: Column): Column = Column.fn("bitmap_or", left, right) /** - * Returns a map whose key-value pairs satisfy a predicate. - * {{{ - * df.select(map_filter(col("m"), (k, v) => k * 10 === v)) - * }}} - * - * @param expr - * the input map column. A column that evaluates to a map. - * @param f - * (key, value) => predicate, the Boolean predicate to filter the input map column + * Returns a bitmap that is the bitwise AND NOT of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * - * @group collection_funcs - * @since 3.0.0 + * @param left + * A column that evaluates to a binary bitmap. + * @param right + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a binary bitmap. */ - def map_filter(expr: Column, f: (Column, Column) => Column): Column = - Column.fn("map_filter", expr, createLambda(f)) + def bitmap_andnot(left: Column, right: Column): Column = + Column.fn("bitmap_andnot", left, right) /** - * Merge two given maps, key-wise into a single map using a function. - * {{{ - * df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)) - * }}} + * Returns a bitmap that is the bitwise XOR of two input bitmaps. The result is always a + * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in + * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise + * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were + * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must + * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar + * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps + * across rows. The representation is not a RoaringBitmap serialization. * * @param left - * the left input map column. A column that evaluates to a map. + * A column that evaluates to a binary bitmap. * @param right - * the right input map column. A column that evaluates to a map. - * @param f - * (key, value1, value2) => new_value, the lambda function to merge the map values - * - * @group collection_funcs - * @since 3.0.0 + * A column that evaluates to a binary bitmap. + * @group misc_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a binary bitmap. */ - def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = - Column.fn("map_zip_with", left, right, createLambda(f)) + def bitmap_xor(left: Column, right: Column): Column = Column.fn("bitmap_xor", left, right) /** - * Returns length of array or map. - * - * This function returns -1 for null input only if spark.sql.ansi.enabled is false and - * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the - * default settings, the function returns null for null input. + * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. The + * input column should be bitmaps created from bitmap_construct_agg(). * - * @param e - * the target column. A column that evaluates to an array or a map. - * @group collection_funcs - * @since 1.5.0 + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def size(e: Column): Column = Column.fn("size", e) + def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) /** - * Returns length of array or map. This is an alias of `size` function. - * - * This function returns -1 for null input only if spark.sql.ansi.enabled is false and - * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the - * default settings, the function returns null for null input. + * Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. The + * input column should be bitmaps created from bitmap_construct_agg(). * - * @param e - * the target column. A column that evaluates to an array or a map. - * @group collection_funcs - * @since 3.5.0 + * @param col + * The input column should be bitmaps created from bitmap_construct_agg(). A column that + * evaluates to a binary. + * @group agg_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a binary. */ - def cardinality(e: Column): Column = Column.fn("cardinality", e) + def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) /** - * Returns a reversed string or an array with reverse order of elements. - * @param e - * the input column. A column that evaluates to a string, a binary, or an array. - * @group collection_funcs - * @since 1.5.0 + * Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. The + * input column should be bitmaps created from bitmap_construct_agg(). + * + * @param col + * A column containing bitmaps created by bitmap_construct_agg() and evaluating to binary + * data. + * @group agg_funcs + * @since 4.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def reverse(e: Column): Column = Column.fn("reverse", e) + def bitmap_xor_agg(col: Column): Column = Column.fn("bitmap_xor_agg", col) ////////////////////////////////////////////////////////////////////////////////////////////// - // Array Functions + // String functions ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Creates a new array column. The input columns must all have the same data type. + * Computes the numeric value of the first character of the string column, and returns the + * result as an int column. * - * @param cols - * The columns to combine into an array. Each is a column of any type, and all must share the - * same data type. - * @group array_funcs - * @since 1.4.0 + * @param e + * The target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - @scala.annotation.varargs - def array(cols: Column*): Column = Column.fn("array", cols: _*) + def ascii(e: Column): Column = Column.fn("ascii", e) /** - * Creates a new array column. The input columns must all have the same data type. + * Computes the BASE64 encoding of a binary column and returns it as a string column. This is + * the reverse of unbase64. * - * @group array_funcs - * @since 1.4.0 + * @param e + * The target column to work on. A column that evaluates to a binary. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def array(colName: String, colNames: String*): Column = { - array((colName +: colNames).map(col): _*) - } + def base64(e: Column): Column = Column.fn("base64", e) /** - * Returns true if the array contains `value`, false if not. Returns null if the array or - * `value` is null, or if `value` is not found and the array contains a null element. - * @param column - * the target column containing the arrays. A column that evaluates to an array. - * @param value - * the value to check for in the array. A column that evaluates to a value matching the - * array's element type. - * @group array_funcs - * @since 1.5.0 + * Computes the BASE32 (RFC 4648) encoding of a binary column and returns it as a string column. + * This is the reverse of from_base32. + * + * @param e + * The target column to work on. A column that evaluates to a binary. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def array_contains(column: Column, value: Any): Column = - Column.fn("array_contains", column, lit(value)) + def to_base32(e: Column): Column = Column.fn("to_base32", e) /** - * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. - * The new element/column is located at end of the ARRAY. + * Calculates the bit length for the specified string column. * - * @param column - * the source column containing the array. A column that evaluates to an array. - * @param element - * the value to append to the array. A column that evaluates to a value matching the array's - * element type. - * @group array_funcs - * @since 3.4.0 + * @param e + * The source column or strings. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def array_append(column: Column, element: Any): Column = - Column.fn("array_append", column, lit(element)) + def bit_length(e: Column): Column = Column.fn("bit_length", e) /** - * Returns `true` if `a1` and `a2` have at least one non-null element in common. If not and both - * the arrays are non-empty and any of them contains a `null`, it returns `null`. It returns - * `false` otherwise. - * @param a1 - * the first input array. A column that evaluates to an array. - * @param a2 - * the second input array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * Concatenates multiple input string columns together into a single string column, using the + * given separator. + * + * @param sep + * The words separator. A column that evaluates to a string. Must be a constant. + * @param exprs + * The list of columns to work on. Each a column that evaluates to a string or an array of + * strings. + * @note + * Input strings which are null are skipped. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) + @scala.annotation.varargs + def concat_ws(sep: String, exprs: Column*): Column = + Column.fn("concat_ws", lit(sep) +: exprs: _*) /** - * Returns an array containing all the elements in `x` from index `start` (or starting from the - * end if `start` is negative) with the specified `length`. - * - * @param x - * the array column to be sliced. A column that evaluates to an array. - * @param start - * the starting index. A column that evaluates to an integer. - * @param length - * the length of the slice. A column that evaluates to an integer. + * Computes the first argument into a string from a binary using the provided character set (one + * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either + * argument is null, the result will also be null. * - * @group array_funcs - * @since 2.4.0 + * @param value + * The target column to work on. A column that evaluates to a binary. + * @param charset + * The charset to use to decode to. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def slice(x: Column, start: Int, length: Int): Column = - slice(x, lit(start), lit(length)) + def decode(value: Column, charset: String): Column = + Column.fn("decode", value, lit(charset)) /** - * Returns an array containing all the elements in `x` from index `start` (or starting from the - * end if `start` is negative) with the specified `length`. - * - * @param x - * the array column to be sliced. A column that evaluates to an array. - * @param start - * the starting index. A column that evaluates to an integer. - * @param length - * the length of the slice. A column that evaluates to an integer. + * Computes the first argument into a binary from a string using the provided character set (one + * of 'US-ASCII', 'ISO-8859-1', 'UTF-8', 'UTF-16BE', 'UTF-16LE', 'UTF-16', 'UTF-32'). If either + * argument is null, the result will also be null. * - * @group array_funcs - * @since 3.1.0 + * @param value + * The target column to work on. A column that evaluates to a string. + * @param charset + * The charset to use to encode. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a binary. */ - def slice(x: Column, start: Column, length: Column): Column = - Column.fn("slice", x, start, length) + def encode(value: Column, charset: String): Column = + Column.fn("encode", value, lit(charset)) /** - * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is - * negative or greater than the number of elements in the array. - * - * @param x - * the array column to be trimmed. A column that evaluates to an array. - * @param n - * the number of elements to remove from the end of the array. Must be between 0 and the - * number of elements in the array (inclusive). + * Returns true if the input is a valid UTF-8 string, otherwise returns false. * - * @group array_funcs - * @since 4.4.0 + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def trim_array(x: Column, n: Int): Column = trim_array(x, lit(n)) + def is_valid_utf8(str: Column): Column = + Column.fn("is_valid_utf8", str) /** - * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is - * negative or greater than the number of elements in the array. - * - * @param x - * the array column to be trimmed. A column that evaluates to an array. - * @param n - * the number of elements to remove from the end of the array. Must be between 0 and the - * number of elements in the array (inclusive). + * Returns a new string in which all invalid UTF-8 byte sequences, if any, are replaced by the + * Unicode replacement character (U+FFFD). * - * @group array_funcs - * @since 4.4.0 + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def trim_array(x: Column, n: Column): Column = Column.fn("trim_array", x, n) + def make_valid_utf8(str: Column): Column = + Column.fn("make_valid_utf8", str) /** - * Concatenates the elements of `column` using the `delimiter`. Null values are replaced with - * `nullReplacement`. - * @param column - * the input column containing the array. A column that evaluates to an array. - * @param delimiter - * the string used to join the array elements. A column that evaluates to a string. - * @param nullReplacement - * the string used to replace null values. A column that evaluates to a string. - * @group array_funcs - * @since 2.4.0 + * Returns the input value if it corresponds to a valid UTF-8 string, or emits a + * SparkIllegalArgumentException exception otherwise. + * + * @param str + * A column of strings, each representing a UTF-8 byte sequence. A column that evaluates to a + * string. + * @group string_funcs + * @since 4.0.0 * @return * Returns a column that evaluates to a string. */ - def array_join(column: Column, delimiter: String, nullReplacement: String): Column = - Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) + def validate_utf8(str: Column): Column = + Column.fn("validate_utf8", str) /** - * Concatenates the elements of `column` using the `delimiter`. - * @param column - * the input column containing the array. A column that evaluates to an array. - * @param delimiter - * the string used to join the array elements. A column that evaluates to a string. - * @group array_funcs - * @since 2.4.0 + * Returns the input value if it corresponds to a valid UTF-8 string, or NULL otherwise. + * + * @param str + * the input value. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return * Returns a column that evaluates to a string. */ - def array_join(column: Column, delimiter: String): Column = - Column.fn("array_join", column, lit(delimiter)) + def try_validate_utf8(str: Column): Column = + Column.fn("try_validate_utf8", str) /** - * Locates the position of the first occurrence of the value in the given array as long. Returns - * null if either of the arguments are null. - * - * @param column - * The array to search. A column that evaluates to an array. - * @param value - * The value to locate. A column. - * @note - * The position is not zero based, but 1 based index. Returns 0 if value could not be found in - * array. + * Returns the Unicode normalization of `str` using the given normalization `form`. Valid forms + * are 'NFC', 'NFD', 'NFKC', and 'NFKD', as defined by Unicode Standard Annex #15. The form name + * is case-insensitive. Normalization is backed by Spark's bundled ICU4J library rather than the + * JVM's own Unicode data, so results are stable across JVM vendors and versions. * - * @group array_funcs - * @since 2.4.0 - * @return - * Returns a column that evaluates to a long. + * @param str + * the input string to normalize. + * @param form + * the normalization form: 'NFC', 'NFD', 'NFKC', or 'NFKD'. + * @group string_funcs + * @since 4.4.0 */ - def array_position(column: Column, value: Any): Column = - Column.fn("array_position", column, lit(value)) + def normalize(str: Column, form: Column): Column = + Column.fn("normalize", str, form) /** - * Returns element of array at given (0-based) index. If the index points outside of the array - * boundaries, then this function returns NULL. + * Returns the Unicode normalization of `str` using the default form 'NFC'. To use a different + * form, call the two-argument overload. * - * @param column - * The array to extract from. A column that evaluates to an array. - * @param index - * The 0-based index. A column that evaluates to an integral. - * @group array_funcs - * @since 3.4.0 - * @return - * Returns a column of the element type of the input array. + * @param str + * the input string to normalize. + * @group string_funcs + * @since 4.4.0 */ - def get(column: Column, index: Column): Column = Column.fn("get", column, index) + def normalize(str: Column): Column = + Column.fn("normalize", str) /** - * Remove all elements that equal to element from the given array. + * Formats numeric column x to a format like '#,###,###.##', rounded to d decimal places with + * HALF_EVEN round mode, and returns the result as a string column. * - * @param column - * The array to remove from. A column that evaluates to an array. - * @param element - * The element to remove. A column. - * @group array_funcs - * @since 2.4.0 + * If d is 0, the result has no decimal point or fractional part. If d is less than 0, the + * result will be null. + * + * @param x + * the numeric value to be formatted. A column that evaluates to a numeric. + * @param d + * the number of decimal places. A column that evaluates to an integral. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def array_remove(column: Column, element: Any): Column = - Column.fn("array_remove", column, lit(element)) + def format_number(x: Column, d: Int): Column = Column.fn("format_number", x, lit(d)) /** - * Remove all null elements from the given array. + * Formats the arguments in printf-style and returns the result as a string column. * - * @param column - * The array to compact. A column that evaluates to an array. - * @group array_funcs - * @since 3.4.0 + * @param format + * the format string that can contain embedded format tags. A column that evaluates to a + * string. Must be a constant. + * @param arguments + * the values to be used in formatting. Columns that evaluate to any type. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def array_compact(column: Column): Column = Column.fn("array_compact", column) + @scala.annotation.varargs + def format_string(format: String, arguments: Column*): Column = + Column.fn("format_string", lit(format) +: arguments: _*) /** - * Returns an array containing value as well as all elements from array. The new element is - * positioned at the beginning of the array. + * Returns a new string column by converting the first letter of each word to uppercase. Words + * are delimited by whitespace. * - * @param column - * The array to prepend to. A column that evaluates to an array. - * @param element - * The element to prepend. A column. - * @group array_funcs - * @since 3.5.0 + * For example, "hello world" will become "Hello World". + * + * @param e + * the target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def array_prepend(column: Column, element: Any): Column = - Column.fn("array_prepend", column, lit(element)) + def initcap(e: Column): Column = Column.fn("initcap", e) /** - * Removes duplicate values from the array. - * @param e - * The array to deduplicate. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * Locate the position of the first occurrence of substr column in the given string. Returns + * null if either of the arguments are null. + * + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. Must be a constant. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def array_distinct(e: Column): Column = Column.fn("array_distinct", e) + def instr(str: Column, substring: String): Column = instr(str, lit(substring)) /** - * Returns an array of the elements in the intersection of the given two arrays, without - * duplicates. + * Locate the position of the first occurrence of substr column in the given string. Returns + * null if either of the arguments are null. * - * @param col1 - * The first array. A column that evaluates to an array. - * @param col2 - * The second array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def array_intersect(col1: Column, col2: Column): Column = - Column.fn("array_intersect", col1, col2) + def instr(str: Column, substring: Column): Column = Column.fn("instr", str, substring) /** - * Adds an item into a given array at a specified position + * Locate the position of the first occurrence of `substring` in `str`, starting the search from + * position `start`. Returns null if either of the arguments are null. * - * @param arr - * The array to insert into. A column that evaluates to an array. - * @param pos - * The 1-based position at which to insert (negative counts from the end). A column that - * evaluates to an integral. - * @param value - * The value to insert. A column. - * @group array_funcs - * @since 3.4.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def array_insert(arr: Column, pos: Column, value: Column): Column = - Column.fn("array_insert", arr, pos, value) + def instr(str: Column, substring: Column, start: Int): Column = + Column.fn("instr", str, substring, lit(start)) /** - * Returns an array of the elements in the union of the given two arrays, without duplicates. + * Locate the position of the first occurrence of `substring` in `str`, starting the search from + * position `start`. Returns null if either of the arguments are null. * - * @param col1 - * The first array. A column that evaluates to an array. - * @param col2 - * The second array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def array_union(col1: Column, col2: Column): Column = - Column.fn("array_union", col1, col2) + def instr(str: Column, substring: Column, start: Column): Column = + Column.fn("instr", str, substring, start) /** - * Returns an array of the elements in the first array but not in the second array, without - * duplicates. The order of elements in the result is not determined + * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the + * search from position `start`. Returns null if either of the arguments are null. * - * @param col1 - * The first array. A column that evaluates to an array. - * @param col2 - * The second array. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. Must be a + * constant. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. Must + * be a constant. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * @note + * The `occurrence` parameter must be a positive integer. + * + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def array_except(col1: Column, col2: Column): Column = - Column.fn("array_except", col1, col2) + def instr(str: Column, substring: Column, start: Int, occurrence: Int): Column = + Column.fn("instr", str, substring, lit(start), lit(occurrence)) /** - * Sorts the input array for the given column in ascending order, according to the natural - * ordering of the array elements. Null elements will be placed at the beginning of the returned - * array. + * Locate the position of the `occurrence`-th occurrence of `substring` in `str`, starting the + * search from position `start`. Returns null if either of the arguments are null. * - * @param e - * the array column to sort. A column that evaluates to an array. - * @group array_funcs - * @since 1.5.0 + * @param str + * the string to search in. A column that evaluates to a string. + * @param substring + * the substring to search for. A column that evaluates to a string. + * @param start + * the position to start the search from. A column that evaluates to an integral. + * @param occurrence + * which occurrence of the substring to locate. A column that evaluates to an integral. + * @note + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. + * @note + * If `start` is positive, the search proceeds forward. If `start` is negative, the search + * proceeds backward from the end of the string. If `start` is 0, returns 0. + * @note + * The `occurrence` parameter must be a positive integer. + * + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def sort_array(e: Column): Column = sort_array(e, asc = true) + def instr(str: Column, substring: Column, start: Column, occurrence: Column): Column = + Column.fn("instr", str, substring, start, occurrence) /** - * Sorts the input array for the given column in ascending or descending order, according to the - * natural ordering of the array elements. NaN is greater than any non-NaN elements for - * double/float type. Null elements will be placed at the beginning of the returned array in - * ascending order or at the end of the returned array in descending order. + * Computes the character length of a given string or number of bytes of a binary string. The + * length of character strings include the trailing spaces. The length of binary strings + * includes binary zeros. * * @param e - * the array column to sort. A column that evaluates to an array. - * @param asc - * whether to sort in ascending order. A column that evaluates to a boolean. Must be a - * constant. - * @group array_funcs + * the target column to work on. A column that evaluates to a string or binary. + * @group string_funcs * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def sort_array(e: Column, asc: Boolean): Column = Column.fn("sort_array", e, lit(asc)) + def length(e: Column): Column = Column.fn("length", e) /** - * Returns the minimum value in the array. NaN is greater than any non-NaN elements for - * double/float type. NULL elements are skipped. + * Computes the character length of a given string or number of bytes of a binary string. The + * length of character strings include the trailing spaces. The length of binary strings + * includes binary zeros. * * @param e - * the array column. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * the target column to work on. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the element type of the input array. + * Returns a column that evaluates to an integer. */ - def array_min(e: Column): Column = Column.fn("array_min", e) + def len(e: Column): Column = Column.fn("len", e) /** - * Returns the maximum value in the array. NaN is greater than any non-NaN elements for - * double/float type. NULL elements are skipped. + * Converts a string column to lower case. * * @param e - * the input column. A column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * the target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.3.0 * @return - * Returns a column of the element type of the input array. + * Returns a column that evaluates to a string. */ - def array_max(e: Column): Column = Column.fn("array_max", e) + def lower(e: Column): Column = Column.fn("lower", e) /** - * Returns the total number of elements in the array. The function returns null for null input. - * - * @param e - * the input column. A column that evaluates to an array. - * @group array_funcs + * Computes the Levenshtein distance of the two given string columns if it's less than or equal + * to a given threshold. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. + * @param threshold + * the maximum distance to compute. A column that evaluates to an integral. Must be a + * constant. + * @return + * result distance, or -1. Returns a column that evaluates to an integer. + * @group string_funcs * @since 3.5.0 + */ + def levenshtein(l: Column, r: Column, threshold: Int): Column = + Column.fn("levenshtein", l, r, lit(threshold)) + + /** + * Computes the Levenshtein distance of the two given string columns. + * @param l + * the first input column. A column that evaluates to a string. + * @param r + * the second input column. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return * Returns a column that evaluates to an integer. */ - def array_size(e: Column): Column = Column.fn("array_size", e) + def levenshtein(l: Column, r: Column): Column = Column.fn("levenshtein", l, r) /** - * Returns a random permutation of the given array. - * - * @param e - * the input column. A column that evaluates to an array. - * @note - * The function is non-deterministic. - * - * @group array_funcs - * @since 2.4.0 + * Computes the Jaro-Winkler similarity between the two given string columns. The result is a + * double between 0.0 (no similarity) and 1.0 (identical). + * @param l + * A column that evaluates to a string. + * @param r + * A column that evaluates to a string. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a double. */ - def shuffle(e: Column): Column = shuffle(e, lit(SparkClassUtils.random.nextLong)) + def jaro_winkler_similarity(l: Column, r: Column): Column = + Column.fn("jaro_winkler_similarity", l, r) /** - * Returns a random permutation of the given array. + * Locate the position of the first occurrence of substr. * - * @param e - * the input column. A column that evaluates to an array. - * @param seed - * the seed for the random generator. A column that evaluates to an integral. Must be a - * constant. + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. * @note - * The function is non-deterministic. + * The position is not zero based, but 1 based index. Returns 0 if substr could not be found + * in str. * - * @group array_funcs - * @since 4.0.0 + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def shuffle(e: Column, seed: Column): Column = Column.fn("shuffle", e, seed) + def locate(substr: String, str: Column): Column = Column.fn("locate", lit(substr), str) /** - * Creates a single array from an array of arrays. If a structure of nested arrays is deeper - * than two levels, only one level of nesting is removed. - * @param e - * the input column. A column that evaluates to an array of arrays. - * @group array_funcs - * @since 2.4.0 + * Locate the position of the first occurrence of substr in a string column, after position pos. + * + * @param substr + * The substring to find. A column that evaluates to a string. + * @param str + * A column that evaluates to a string. + * @param pos + * The starting position. A column that evaluates to an integer. + * @note + * The position is not zero based, but 1 based index. returns 0 if substr could not be found + * in str. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def flatten(e: Column): Column = Column.fn("flatten", e) + def locate(substr: String, str: Column, pos: Int): Column = + Column.fn("locate", lit(substr), str, lit(pos)) /** - * Generate a sequence of integers from start to stop, incrementing by step. + * Left-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * @param start - * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a - * date, or a timestamp. - * @param stop - * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, - * or a timestamp. - * @param step - * the value to add to the current element to get the next element. A column that evaluates to - * an integral or interval. - * @group array_funcs - * @since 2.4.0 + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def sequence(start: Column, stop: Column, step: Column): Column = - Column.fn("sequence", start, stop, step) + def lpad(str: Column, len: Int, pad: String): Column = lpad(str, lit(len), lit(pad)) /** - * Generate a sequence of integers from start to stop, incrementing by 1 if start is less than - * or equal to stop, otherwise -1. + * Left-pad the binary column with pad to a byte length of len. If the binary column is longer + * than len, the return value is shortened to len bytes. * - * @param start - * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a - * date, or a timestamp. - * @param stop - * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, - * or a timestamp. - * @group array_funcs - * @since 2.4.0 + * @param str + * A column that evaluates to a binary. + * @param len + * The byte length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding bytes. A column that evaluates to a binary. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def sequence(start: Column, stop: Column): Column = Column.fn("sequence", start, stop) + def lpad(str: Column, len: Int, pad: Array[Byte]): Column = lpad(str, lit(len), lit(pad)) /** - * Creates an array containing the left argument repeated the number of times given by the right - * argument. + * Left-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * @param left - * the value to repeat. A column that evaluates to any type. - * @param right - * the number of times to repeat the value. A column that evaluates to an integral. - * @group array_funcs - * @since 2.4.0 + * @param str + * A column that evaluates to a string. + * @param len + * The length of the padded result. A column that evaluates to an integer. + * @param pad + * The padding string. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def array_repeat(left: Column, right: Column): Column = Column.fn("array_repeat", left, right) + def lpad(str: Column, len: Column, pad: Column): Column = Column.fn("lpad", str, len, pad) /** - * Creates an array containing the left argument repeated the number of times given by the right - * argument. + * Trim the spaces from left end for the specified string value. * * @param e - * the value to repeat. A column that evaluates to any type. - * @param count - * the number of times to repeat the value. A column that evaluates to an integral. Must be a - * constant. - * @group array_funcs - * @since 2.4.0 + * A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def array_repeat(e: Column, count: Int): Column = array_repeat(e, lit(count)) + def ltrim(e: Column): Column = Column.fn("ltrim", e) /** - * Returns a merged array of structs in which the N-th struct contains all N-th values of input - * arrays. + * Trim the specified character string from left end for the specified string column. * @param e - * the columns of arrays to be merged. Each is a column that evaluates to an array. - * @group array_funcs - * @since 2.4.0 + * A column that evaluates to a string. + * @param trimString + * The trim string. A column that evaluates to a string. + * @group string_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def arrays_zip(e: Column*): Column = Column.fn("arrays_zip", e: _*) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Struct Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def ltrim(e: Column, trimString: String): Column = ltrim(e, lit(trimString)) /** - * Creates a struct with the given field names and values. - * - * @param cols - * The field names and values grouped as pairs (name1, value1, name2, value2, ...). Names are - * columns that evaluate to a string; values are columns of any type. - * @group struct_funcs - * @since 3.5.0 + * Trim the specified character string from left end for the specified string column. + * @param e + * A column that evaluates to a string. + * @param trim + * The trim string. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def named_struct(cols: Column*): Column = Column.fn("named_struct", cols: _*) + def ltrim(e: Column, trim: Column): Column = Column.fn("ltrim", trim, e) /** - * Creates a new struct column. If the input column is a column in a `DataFrame`, or a derived - * column expression that is named (i.e. aliased), its name would be retained as the - * StructField's name, otherwise, the newly generated StructField's name would be auto generated - * as `col` with a suffix `index + 1`, i.e. col1, col2, col3, ... + * Calculates the byte length for the specified string column. * - * @param cols - * the columns to contain in the output struct. A column of any type. - * @group struct_funcs - * @since 1.4.0 + * @param e + * A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to an integer. */ - @scala.annotation.varargs - def struct(cols: Column*): Column = Column.fn("struct", cols: _*) + def octet_length(e: Column): Column = Column.fn("octet_length", e) /** - * Creates a new struct column that composes multiple input columns. + * Marks a given column with specified collation. * - * @param colName - * the name of the first column to contain in the output struct. - * @param colNames - * the names of the remaining columns to contain in the output struct. - * @group struct_funcs - * @since 1.4.0 + * @param e + * A column that evaluates to a string. + * @param collation + * The collation name. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def struct(colName: String, colNames: String*): Column = { - struct((colName +: colNames).map(col): _*) - } - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Map Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def collate(e: Column, collation: String): Column = Column.fn("collate", e, lit(collation)) /** - * Creates a new map column. The input columns must be grouped as key-value pairs, e.g. (key1, - * value1, key2, value2, ...). The key columns must all have the same data type, and can't be - * null. The value columns must all have the same data type. + * Returns the collation name of a given column. * - * @param cols - * The columns grouped as key-value pairs (key1, value1, key2, value2, ...). Each is a column - * of any type; key columns must share a type and value columns must share a type. - * @group map_funcs - * @since 2.0 + * @param e + * A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def map(cols: Column*): Column = Column.fn("map", cols: _*) + def collation(e: Column): Column = Column.fn("collation", e) /** - * Creates a new map column. The array in the first column is used for keys. The array in the - * second column is used for values. All elements in the array for key should not be null. + * Returns true if `str` matches `regexp`, or false otherwise. * - * @param keys - * The array of keys for the map; elements must not be null. A column that evaluates to an - * array. - * @param values - * The array of values for the map. A column that evaluates to an array. - * @group map_funcs - * @since 2.4 + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a boolean. */ - def map_from_arrays(keys: Column, values: Column): Column = - Column.fn("map_from_arrays", keys, values) + def rlike(str: Column, regexp: Column): Column = Column.fn("rlike", str, regexp) /** - * Creates a map after splitting the text into key/value pairs using delimiters. Both - * `pairDelim` and `keyValueDelim` are treated as regular expressions. + * Returns true if `str` matches `regexp`, or false otherwise. * - * @param text - * The text to split into key/value pairs. A column that evaluates to a string. - * @param pairDelim - * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to - * a string. - * @param keyValueDelim - * Delimiter used to split key and value, treated as a regular expression. A column that - * evaluates to a string. - * @group map_funcs + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. + * @group predicate_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a boolean. */ - def str_to_map(text: Column, pairDelim: Column, keyValueDelim: Column): Column = - Column.fn("str_to_map", text, pairDelim, keyValueDelim) + def regexp(str: Column, regexp: Column): Column = Column.fn("regexp", str, regexp) /** - * Creates a map after splitting the text into key/value pairs using delimiters. The `pairDelim` - * is treated as regular expressions. + * Returns true if `str` matches `regexp`, or false otherwise. * - * @param text - * The text to split into key/value pairs. A column that evaluates to a string. - * @param pairDelim - * Delimiter used to split pairs, treated as a regular expression. A column that evaluates to - * a string. - * @group map_funcs + * @param str + * A column that evaluates to a string. + * @param regexp + * The regular expression pattern. A column that evaluates to a string. + * @group predicate_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a boolean. */ - def str_to_map(text: Column, pairDelim: Column): Column = - Column.fn("str_to_map", text, pairDelim) + def regexp_like(str: Column, regexp: Column): Column = Column.fn("regexp_like", str, regexp) /** - * Creates a map after splitting the text into key/value pairs using delimiters. + * Returns a count of the number of times that the regular expression pattern `regexp` is + * matched in the string `str`. * - * @param text - * The text to split into key/value pairs. A column that evaluates to a string. - * @group map_funcs + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to an integer. */ - def str_to_map(text: Column): Column = Column.fn("str_to_map", text) + def regexp_count(str: Column, regexp: Column): Column = Column.fn("regexp_count", str, regexp) /** - * Returns true if the map contains the key. - * @param column - * the input column. A column that evaluates to a map. - * @param key - * the key to check for. A column that evaluates to the map's key type. Must be a constant. - * @group map_funcs - * @since 3.3.0 + * Extract a specific group matched by a Java regex, from the specified string column. If the + * regex did not match, or the specified group did not match, an empty string is returned. if + * the specified group index exceeds the group count of regex, an IllegalArgumentException will + * be thrown. + * + * @param e + * target column to work on. A column that evaluates to a string. + * @param exp + * regex pattern to apply. A string. Must be a constant. + * @param groupIdx + * matched group id. An integer. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a string. */ - def map_contains_key(column: Column, key: Any): Column = - Column.fn("map_contains_key", column, lit(key)) + def regexp_extract(e: Column, exp: String, groupIdx: Int): Column = + Column.fn("regexp_extract", e, lit(exp), lit(groupIdx)) /** - * Returns an unordered array containing the keys of the map. - * @param e - * the input column. A column that evaluates to a map. - * @group map_funcs - * @since 2.3.0 + * Extract all strings in the `str` that match the `regexp` expression and corresponding to the + * first regex group index. + * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to an array. */ - def map_keys(e: Column): Column = Column.fn("map_keys", e) + def regexp_extract_all(str: Column, regexp: Column): Column = + Column.fn("regexp_extract_all", str, regexp) /** - * Returns an unordered array containing the values of the map. - * @param e - * the input column. A column that evaluates to a map. - * @group map_funcs - * @since 2.3.0 + * Extract all strings in the `str` that match the `regexp` expression and corresponding to the + * regex group index. + * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. + * @group string_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to an array. */ - def map_values(e: Column): Column = Column.fn("map_values", e) + def regexp_extract_all(str: Column, regexp: Column, idx: Column): Column = + Column.fn("regexp_extract_all", str, regexp, idx) /** - * Returns an unordered array of all entries in the given map. + * Replace all substrings of the specified string value that match regexp with rep. + * * @param e - * the input column. A column that evaluates to a map. - * @group map_funcs - * @since 3.0.0 + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def map_entries(e: Column): Column = Column.fn("map_entries", e) + def regexp_replace(e: Column, pattern: String, replacement: String): Column = + regexp_replace(e, lit(pattern), lit(replacement)) /** - * Returns a map created from the given array of entries. + * Replace all substrings of the specified string value that match regexp with rep, starting at + * the specified position `pos`. + * * @param e - * the array of entries to convert. A column that evaluates to an array of structs, each with - * a key and value field. - * @group map_funcs - * @since 2.4.0 + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A string. Must be a constant. + * @param replacement + * replacement string. A string. Must be a constant. + * @param pos + * position to start replacement. The first position is 1. An integer. Must be a constant. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a string. */ - def map_from_entries(e: Column): Column = Column.fn("map_from_entries", e) + def regexp_replace(e: Column, pattern: String, replacement: String, pos: Int): Column = + regexp_replace(e, lit(pattern), lit(replacement), lit(pos)) /** - * Returns the union of all the given maps. - * @param cols - * the maps to merge. Each is a column that evaluates to a map. - * @group map_funcs - * @since 2.4.0 + * Replace all substrings of the specified string value that match regexp with rep. + * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. + * @group string_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a map. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def map_concat(cols: Column*): Column = Column.fn("map_concat", cols: _*) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Aggregate Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def regexp_replace(e: Column, pattern: Column, replacement: Column): Column = + Column.fn("regexp_replace", e, pattern, replacement) /** - * @group agg_funcs - * @since 1.3.0 + * Replace all substrings of the specified string value that match regexp with rep, starting at + * the specified position `pos`. + * + * @param e + * target column to work on. A column that evaluates to a string. + * @param pattern + * regex pattern to apply. A column that evaluates to a string. + * @param replacement + * replacement string. A column that evaluates to a string. + * @param pos + * position to start replacement. The first position is 1. A column that evaluates to an + * integer. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(e: Column): Column = approx_count_distinct(e) + def regexp_replace(e: Column, pattern: Column, replacement: Column, pos: Column): Column = + Column.fn("regexp_replace", e, pattern, replacement, pos) /** - * @group agg_funcs - * @since 1.3.0 + * Returns the substring that matches the regular expression `regexp` within the string `str`. + * If the regular expression is not found, the result is null. + * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(columnName: String): Column = approx_count_distinct(columnName) + def regexp_substr(str: Column, regexp: Column): Column = Column.fn("regexp_substr", str, regexp) /** - * @group agg_funcs - * @since 1.3.0 + * Searches a string for a regular expression and returns an integer that indicates the + * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match + * is found, returns 0. + * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an integer. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(e: Column, rsd: Double): Column = approx_count_distinct(e, rsd) + def regexp_instr(str: Column, regexp: Column): Column = Column.fn("regexp_instr", str, regexp) /** - * @group agg_funcs - * @since 1.3.0 + * Searches a string for a regular expression and returns an integer that indicates the + * beginning position of the matched substring. Positions are 1-based, not 0-based. If no match + * is found, returns 0. + * + * @param str + * target column to work on. A column that evaluates to a string. + * @param regexp + * regex pattern to apply. A column that evaluates to a string. + * @param idx + * matched group id. A column that evaluates to an integer. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an integer. */ - @deprecated("Use approx_count_distinct", "2.1.0") - def approxCountDistinct(columnName: String, rsd: Double): Column = { - approx_count_distinct(Column(columnName), rsd) - } + def regexp_instr(str: Column, regexp: Column, idx: Column): Column = + Column.fn("regexp_instr", str, regexp, idx) /** - * Aggregate function: returns the approximate number of distinct items in a group. + * Decodes a BASE64 encoded string column and returns it as a binary column. This is the reverse + * of base64. * * @param e - * The column to count distinct values in. A column of any type. - * @group agg_funcs - * @since 2.1.0 + * target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def approx_count_distinct(e: Column): Column = Column.fn("approx_count_distinct", e) + def unbase64(e: Column): Column = Column.fn("unbase64", e) /** - * Aggregate function: returns the approximate number of distinct items in a group. + * Decodes a BASE32 (RFC 4648) encoded string column and returns it as a binary column. This is + * the reverse of to_base32. * - * @param columnName - * The name of the column to count distinct values in. A column of any type. - * @group agg_funcs - * @since 2.1.0 + * @param e + * target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a binary. */ - def approx_count_distinct(columnName: String): Column = approx_count_distinct( - column(columnName)) + def from_base32(e: Column): Column = Column.fn("from_base32", e) /** - * Aggregate function: returns the approximate number of distinct items in a group. - * - * @param rsd - * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a - * double. Must be a constant. + * Right-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * @group agg_funcs - * @since 2.1.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param len + * length of the final string. An integer. Must be a constant. + * @param pad + * chars to append. A string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def approx_count_distinct(e: Column, rsd: Double): Column = { - Column.fn("approx_count_distinct", e, lit(rsd)) - } + def rpad(str: Column, len: Int, pad: String): Column = rpad(str, lit(len), lit(pad)) /** - * Aggregate function: returns the approximate number of distinct items in a group. - * - * @param rsd - * maximum relative standard deviation allowed (default = 0.05). A column that evaluates to a - * double. Must be a constant. + * Right-pad the binary column with pad to a byte length of len. If the binary column is longer + * than len, the return value is shortened to len bytes. * - * @group agg_funcs - * @since 2.1.0 + * @param str + * target column to work on. A column that evaluates to a binary. + * @param len + * byte length of the final binary. An integer. Must be a constant. + * @param pad + * bytes to append. A binary. Must be a constant. + * @group string_funcs + * @since 3.3.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def approx_count_distinct(columnName: String, rsd: Double): Column = { - approx_count_distinct(Column(columnName), rsd) - } + def rpad(str: Column, len: Int, pad: Array[Byte]): Column = rpad(str, lit(len), lit(pad)) /** - * Aggregate function: returns the average of the values in a group. + * Right-pad the string column with pad to a length of len. If the string column is longer than + * len, the return value is shortened to len characters. * - * @param e - * The column to average. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 1.3.0 + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param len + * length of the final result. A column that evaluates to an integer. + * @param pad + * chars or bytes to append. A column that evaluates to a string or binary. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a numeric. + * Returns a column of the same type as the input. */ - def avg(e: Column): Column = Column.fn("avg", e) + def rpad(str: Column, len: Column, pad: Column): Column = Column.fn("rpad", str, len, pad) /** - * Aggregate function: returns the average of the values in a group. + * Repeats a string column n times, and returns it as a new string column. * - * @param columnName - * The name of the column to average. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 1.3.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. Must be a + * constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a numeric. + * Returns a column that evaluates to a string. */ - def avg(columnName: String): Column = avg(Column(columnName)) + def repeat(str: Column, n: Int): Column = Column.fn("repeat", str, lit(n)) /** - * Aggregate function: returns a list of objects with duplicates. - * - * @param e - * The column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Repeats a string column n times, and returns it as a new string column. * - * @group agg_funcs - * @since 1.6.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param n + * number of times to repeat value. A column that evaluates to an integral. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def collect_list(e: Column): Column = Column.fn("collect_list", e) + def repeat(str: Column, n: Column): Column = Column.fn("repeat", str, n) /** - * Aggregate function: returns a list of objects with duplicates. - * - * @param columnName - * The name of the column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * Trim the spaces from right end for the specified string value. * - * @group agg_funcs - * @since 1.6.0 + * @param e + * target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def collect_list(columnName: String): Column = collect_list(Column(columnName)) + def rtrim(e: Column): Column = Column.fn("rtrim", e) /** - * Aggregate function: returns a set of objects with duplicate elements eliminated. - * + * Trim the specified character string from right end for the specified string column. * @param e - * The column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 1.6.0 + * target column to work on. A column that evaluates to a string. + * @param trimString + * the trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def collect_set(e: Column): Column = Column.fn("collect_set", e) + def rtrim(e: Column, trimString: String): Column = rtrim(e, lit(trimString)) /** - * Aggregate function: returns a set of objects with duplicate elements eliminated. - * - * @param columnName - * The name of the column to collect. A column of any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 1.6.0 + * Trim the specified character string from right end for the specified string column. + * @param e + * target column to work on. A column that evaluates to a string. + * @param trim + * the trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def collect_set(columnName: String): Column = collect_set(Column(columnName)) + def rtrim(e: Column, trim: Column): Column = Column.fn("rtrim", trim, e) /** - * Aggregate function: returns the distinct union of the elements of an array-typed column - * across rows. - * - * The aggregation buffer holds only the distinct elements, so its size is bounded by the - * element universe rather than by the number of input rows. Null elements are dropped by - * default (IGNORE NULLS), matching `collect_set`. With `RESPECT NULLS`, a single null element - * is kept, in which case this is equivalent to `array_distinct(flatten(collect_list(e)))`. The - * `RESPECT NULLS` clause is only available through SQL (e.g. - * `expr("collect_union(col) RESPECT NULLS")`). + * Returns the soundex code for the specified expression. * * @param e - * The array column to collect the union of. A column of type array. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 4.3.0 + * target column to work on. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def collect_union(e: Column): Column = Column.fn("collect_union", e) + def soundex(e: Column): Column = Column.fn("soundex", e) /** - * Aggregate function: returns the distinct union of the elements of an array-typed column - * across rows. + * Splits str around matches of the given pattern. * - * @param columnName - * The name of the array column to collect the union of. A column of type array. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a string representing a regular expression. The regex string should be a Java regular + * expression. A column that evaluates to a string. * - * @group agg_funcs - * @since 4.3.0 + * @group string_funcs + * @since 1.5.0 * @return * Returns a column that evaluates to an array. */ - def collect_union(columnName: String): Column = collect_union(Column(columnName)) + def split(str: Column, pattern: String): Column = Column.fn("split", str, lit(pattern)) /** - * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is - * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min - * sketch is a probabilistic data structure used for cardinality estimation using sub-linear - * space. + * Splits str around matches of the given pattern. * - * @param e - * The column to compute the sketch on. A column that evaluates to an integral, string or - * binary. - * @param eps - * The relative error, must be positive. A column that evaluates to a numeric. Must be a - * constant. - * @param confidence - * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. - * Must be a constant. - * @param seed - * The random seed. A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a column of string representing a regular expression. The regex string should be a Java + * regular expression. A column that evaluates to a string. + * + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def count_min_sketch(e: Column, eps: Column, confidence: Column, seed: Column): Column = - Column.fn("count_min_sketch", e, eps, confidence, seed) + def split(str: Column, pattern: Column): Column = Column.fn("split", str, pattern) /** - * Returns a count-min sketch of a column with the given esp, confidence and seed. The result is - * an array of bytes, which can be deserialized to a `CountMinSketch` before usage. Count-min - * sketch is a probabilistic data structure used for cardinality estimation using sub-linear - * space. + * Splits str around matches of the given pattern. * - * @param e - * The column to compute the sketch on. A column that evaluates to an integral, string or - * binary. - * @param eps - * The relative error, must be positive. A column that evaluates to a numeric. Must be a - * constant. - * @param confidence - * The confidence, must be positive and less than 1.0. A column that evaluates to a numeric. - * Must be a constant. - * @group agg_funcs - * @since 4.0.0 + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a string representing a regular expression. The regex string should be a Java regular + * expression. A column that evaluates to a string. + * @param limit + * an integer expression which controls the number of times the regex is applied.
    + *
  • limit greater than 0: The resulting array's length will not be more than limit, and the + * resulting array's last entry will contain all input beyond the last matched regex.
    • + *
  • limit less than or equal to 0: `regex` will be applied as many times as possible, and + * the resulting array can be of any size.
A column that evaluates to an integer. + * + * @group string_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def count_min_sketch(e: Column, eps: Column, confidence: Column): Column = - count_min_sketch(e, eps, confidence, lit(SparkClassUtils.random.nextLong)) + def split(str: Column, pattern: String, limit: Int): Column = + Column.fn("split", str, lit(pattern), lit(limit)) /** - * Aggregate function: returns the Pearson Correlation Coefficient for two columns. + * Splits str around matches of the given pattern. * - * @param column1 - * The first column. A column that evaluates to a numeric. - * @param column2 - * The second column. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param str + * a string expression to split. A column that evaluates to a string. + * @param pattern + * a column of string representing a regular expression. The regex string should be a Java + * regular expression. A column that evaluates to a string. + * @param limit + * a column of integer expression which controls the number of times the regex is applied. + *
  • limit greater than 0: The resulting array's length will not be more than limit, + * and the resulting array's last entry will contain all input beyond the last matched + * regex.
  • limit less than or equal to 0: `regex` will be applied as many times as + * possible, and the resulting array can be of any size.
A column that evaluates to + * an integer. + * + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def corr(column1: Column, column2: Column): Column = Column.fn("corr", column1, column2) + def split(str: Column, pattern: Column, limit: Column): Column = + Column.fn("split", str, pattern, limit) /** - * Aggregate function: returns the Pearson Correlation Coefficient for two columns. + * Substring starts at `pos` and is of length `len` when str is String type or returns the slice + * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * - * @param columnName1 - * The name of the first column. A column that evaluates to a numeric. - * @param columnName2 - * The name of the second column. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. Must be a constant. + * @param len + * length of chars. A column that evaluates to an integral. Must be a constant. + * @note + * The position is not zero based, but 1 based index. + * + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column of the same type as the input. */ - def corr(columnName1: String, columnName2: String): Column = { - corr(Column(columnName1), Column(columnName2)) - } + def substring(str: Column, pos: Int, len: Int): Column = + Column.fn("substring", str, lit(pos), lit(len)) /** - * Aggregate function: returns the number of items in a group. + * Substring starts at `pos` and is of length `len` when str is String type or returns the slice + * of byte array that starts at `pos` in byte and is of length `len` when str is Binary type * - * @param e - * The column to count. A column of any type. - * @group agg_funcs - * @since 1.3.0 + * @param str + * target column to work on. A column that evaluates to a string or binary. + * @param pos + * starting position in str. A column that evaluates to an integral. + * @param len + * length of chars. A column that evaluates to an integral. + * @note + * The position is not zero based, but 1 based index. + * + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - def count(e: Column): Column = - Column.fn("count", e) + def substring(str: Column, pos: Column, len: Column): Column = + Column.fn("substring", str, pos, len) /** - * Aggregate function: returns the number of items in a group. + * Returns the substring from string str before count occurrences of the delimiter delim. If + * count is positive, everything the left of the final delimiter (counting from left) is + * returned. If count is negative, every to the right of the final delimiter (counting from the + * right) is returned. substring_index performs a case-sensitive match when searching for delim. * - * @param columnName - * The name of the column to count. A column of any type. - * @group agg_funcs - * @since 1.3.0 + * @param str + * target column to work on. A column that evaluates to a string. + * @param delim + * delimiter of values. A column that evaluates to a string. Must be a constant. + * @param count + * number of occurrences. A column that evaluates to an integral. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def count(columnName: String): TypedColumn[Any, Long] = - count(Column(columnName)).as(PrimitiveLongEncoder) + def substring_index(str: Column, delim: String, count: Int): Column = + Column.fn("substring_index", str, lit(delim), lit(count)) /** - * Aggregate function: returns the number of distinct items in a group. - * - * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. + * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of + * `src` and proceeding for `len` bytes. * - * @param expr - * The first column. A column of any type. - * @param exprs - * Additional columns. A column of any type. - * @group agg_funcs - * @since 1.3.0 + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. + * @param len + * the number of bytes to replace in src. A column that evaluates to an integral. + * @group string_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the input. */ - @scala.annotation.varargs - def countDistinct(expr: Column, exprs: Column*): Column = count_distinct(expr, exprs: _*) + def overlay(src: Column, replace: Column, pos: Column, len: Column): Column = + Column.fn("overlay", src, replace, pos, len) /** - * Aggregate function: returns the number of distinct items in a group. - * - * An alias of `count_distinct`, and it is encouraged to use `count_distinct` directly. + * Overlay the specified portion of `src` with `replace`, starting from byte position `pos` of + * `src`. * - * @param columnName - * first column to compute on. A column of any type. - * @param columnNames - * additional columns to compute on. Columns of any type. - * @group agg_funcs - * @since 1.3.0 + * @param src + * the string that will be replaced. A column that evaluates to a string or binary. + * @param replace + * the substitution string. A column that evaluates to a string or binary. + * @param pos + * the starting position in src. A column that evaluates to an integral. + * @group string_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def countDistinct(columnName: String, columnNames: String*): Column = - count_distinct(Column(columnName), columnNames.map(Column.apply): _*) + def overlay(src: Column, replace: Column, pos: Column): Column = + Column.fn("overlay", src, replace, pos) /** - * Aggregate function: returns the number of distinct items in a group. - * - * @param expr - * first column to compute on. A column of any type. - * @param exprs - * additional columns to compute on. Columns of any type. - * @group agg_funcs + * Splits a string into arrays of sentences, where each sentence is an array of words. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. + * @param country + * a country of the locale. A column that evaluates to a string. + * @group string_funcs * @since 3.2.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an array. */ - @scala.annotation.varargs - def count_distinct(expr: Column, exprs: Column*): Column = - Column.fn("count", isDistinct = true, expr +: exprs: _*) + def sentences(string: Column, language: Column, country: Column): Column = + Column.fn("sentences", string, language, country) /** - * Aggregate function: returns the population covariance for two columns. - * - * @param column1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param column2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * Splits a string into arrays of sentences, where each sentence is an array of words. The + * default `country`('') is used. + * @param string + * a string to be split. A column that evaluates to a string. + * @param language + * a language of the locale. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def covar_pop(column1: Column, column2: Column): Column = - Column.fn("covar_pop", column1, column2) + def sentences(string: Column, language: Column): Column = + Column.fn("sentences", string, language) /** - * Aggregate function: returns the population covariance for two columns. - * - * @param columnName1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param columnName2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * Splits a string into arrays of sentences, where each sentence is an array of words. The + * default locale is used. + * @param string + * a string to be split. A column that evaluates to a string. + * @group string_funcs + * @since 3.2.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def covar_pop(columnName1: String, columnName2: String): Column = { - covar_pop(Column(columnName1), Column(columnName2)) - } + def sentences(string: Column): Column = Column.fn("sentences", string) /** - * Aggregate function: returns the sample covariance for two columns. + * Translate any character in the src by a character in replaceString. The characters in + * replaceString correspond to the characters in matchingString. The translate will happen when + * any character in the string matches the character in the `matchingString`. * - * @param column1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param column2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * @param src + * source column to work on. A column that evaluates to a string. + * @param matchingString + * matching characters. A column that evaluates to a string. Must be a constant. + * @param replaceString + * characters for replacement. A column that evaluates to a string. Must be a constant. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def covar_samp(column1: Column, column2: Column): Column = - Column.fn("covar_samp", column1, column2) + def translate(src: Column, matchingString: String, replaceString: String): Column = + Column.fn("translate", src, lit(matchingString), lit(replaceString)) /** - * Aggregate function: returns the sample covariance for two columns. + * Trim the spaces from both ends for the specified string column. * - * @param columnName1 - * first column to calculate covariance. A column that evaluates to a numeric. - * @param columnName2 - * second column to calculate covariance. A column that evaluates to a numeric. - * @group agg_funcs - * @since 2.0.0 + * @param e + * The string column to trim. A column that evaluates to a string. + * @group string_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def covar_samp(columnName1: String, columnName2: String): Column = { - covar_samp(Column(columnName1), Column(columnName2)) - } + def trim(e: Column): Column = Column.fn("trim", e) /** - * Aggregate function: returns the first value in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * + * Trim the specified character from both ends for the specified string column. * @param e - * column to fetch the first value for. A column of any type. - * @param ignoreNulls - * if first value is null then look for first non-null value. A column that evaluates to a - * boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 2.0.0 + * The string column to trim. A column that evaluates to a string. + * @param trimString + * The trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 2.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def first(e: Column, ignoreNulls: Boolean): Column = - Column.fn("first", false, e, lit(ignoreNulls)) + def trim(e: Column, trimString: String): Column = trim(e, lit(trimString)) /** - * Aggregate function: returns the first value of a column in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param columnName - * column to fetch the first value for. A column of any type. - * @param ignoreNulls - * if first value is null then look for first non-null value. A column that evaluates to a - * boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 2.0.0 + * Trim the specified character from both ends for the specified string column. + * @param e + * The string column to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 4.0.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def first(columnName: String, ignoreNulls: Boolean): Column = { - first(Column(columnName), ignoreNulls) - } + def trim(e: Column, trim: Column): Column = Column.fn("trim", trim, e) /** - * Aggregate function: returns the first value in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * Converts a string column to upper case. * * @param e - * column to fetch the first value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs + * The input column to convert to upper case. A column that evaluates to a string. + * @group string_funcs * @since 1.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def first(e: Column): Column = first(e, ignoreNulls = false) + def upper(e: Column): Column = Column.fn("upper", e) /** - * Aggregate function: returns the first value of a column in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param columnName - * column to fetch the first value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Converts the input `e` to a binary value based on the supplied `format`. The `format` can be + * a case-insensitive string literal of "hex", "utf-8", "utf8", or "base64". By default, the + * binary format for conversion is "hex" if `format` is omitted. The function returns NULL if at + * least one of the input parameters is NULL. * - * @group agg_funcs - * @since 1.3.0 + * @param e + * The input value to convert. A column that evaluates to a string. + * @param f + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def first(columnName: String): Column = first(Column(columnName)) + def to_binary(e: Column, f: Column): Column = Column.fn("to_binary", e, f) /** - * Aggregate function: returns the first value in a group. + * Converts the input `e` to a binary value based on the default format "hex". The function + * returns NULL if at least one of the input parameters is NULL. * * @param e - * column to fetch the first value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs + * The input value to convert. A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def first_value(e: Column): Column = Column.fn("first_value", e) + def to_binary(e: Column): Column = Column.fn("to_binary", e) + // scalastyle:off line.size.limit /** - * Aggregate function: returns the first value in a group. - * - * The function by default returns the first values it sees. It will return the first non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * column to fetch the first value for. A column of any type. - * @param ignoreNulls - * if first value is null then look for first non-null value. A column that evaluates to a - * boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. + * The format can consist of the following characters, case insensitive: '0' or '9': Specifies + * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a + * sequence of digits in the input value, generating a result string of the same length as the + * corresponding sequence in the format string. The result string is left-padded with zeros if + * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts + * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': + * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': + * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to + * the left and right of each grouping separator. '$': Specifies the location of the $ currency + * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' + * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note + * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the + * end of the format string; specifies that the result string will be wrapped by angle brackets + * if the input value is negative. * - * @group agg_funcs + * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime + * Patterns. If `e` is a binary, it is converted to a string in one of the formats: + * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input + * binary is decoded to UTF-8 string. + * + * @param e + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant when `e` is a numeric or binary value. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def first_value(e: Column, ignoreNulls: Column): Column = - Column.fn("first_value", e, ignoreNulls) + // scalastyle:on line.size.limit + def to_char(e: Column, format: Column): Column = Column.fn("to_char", e, format) + // scalastyle:off line.size.limit /** - * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or - * not, returns 1 for aggregated or 0 for not aggregated in the result set. + * Convert `e` to a string based on the `format`. Throws an exception if the conversion fails. + * The format can consist of the following characters, case insensitive: '0' or '9': Specifies + * an expected digit between 0 and 9. A sequence of 0 or 9 in the format string matches a + * sequence of digits in the input value, generating a result string of the same length as the + * corresponding sequence in the format string. The result string is left-padded with zeros if + * the 0/9 sequence comprises more digits than the matching part of the decimal value, starts + * with 0, and is before the decimal point. Otherwise, it is padded with spaces. '.' or 'D': + * Specifies the position of the decimal point (optional, only allowed once). ',' or 'G': + * Specifies the position of the grouping (thousands) separator (,). There must be a 0 or 9 to + * the left and right of each grouping separator. '$': Specifies the location of the $ currency + * sign. This character may only be specified once. 'S' or 'MI': Specifies the position of a '-' + * or '+' sign (optional, only allowed once at the beginning or end of the format string). Note + * that 'S' prints '+' for positive values but 'MI' prints a space. 'PR': Only allowed at the + * end of the format string; specifies that the result string will be wrapped by angle brackets + * if the input value is negative. + * + * If `e` is a datetime, `format` shall be a valid datetime pattern, see Datetime + * Patterns. If `e` is a binary, it is converted to a string in one of the formats: + * 'base64': a base 64 string. 'hex': a string in the hexadecimal format. 'utf-8': the input + * binary is decoded to UTF-8 string. * * @param e - * column to check if it is aggregated. A column of any type. - * @group agg_funcs - * @since 2.0.0 + * The input value to convert. A column that evaluates to a numeric, date, timestamp or + * binary. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant when `e` is a numeric or binary value. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a byte. + * Returns a column that evaluates to a string. */ - def grouping(e: Column): Column = Column.fn("grouping", e) + // scalastyle:on line.size.limit + def to_varchar(e: Column, format: Column): Column = Column.fn("to_varchar", e, format) /** - * Aggregate function: indicates whether a specified column in a GROUP BY list is aggregated or - * not, returns 1 for aggregated or 0 for not aggregated in the result set. + * Convert string 'e' to a number based on the string format 'format'. Throws an exception if + * the conversion fails. The format can consist of the following characters, case insensitive: + * '0' or '9': Specifies an expected digit between 0 and 9. A sequence of 0 or 9 in the format + * string matches a sequence of digits in the input string. If the 0/9 sequence starts with 0 + * and is before the decimal point, it can only match a digit sequence of the same size. + * Otherwise, if the sequence starts with 9 or is after the decimal point, it can match a digit + * sequence that has the same or smaller size. '.' or 'D': Specifies the position of the decimal + * point (optional, only allowed once). ',' or 'G': Specifies the position of the grouping + * (thousands) separator (,). There must be a 0 or 9 to the left and right of each grouping + * separator. 'expr' must match the grouping separator relevant for the size of the number. '$': + * Specifies the location of the $ currency sign. This character may only be specified once. 'S' + * or 'MI': Specifies the position of a '-' or '+' sign (optional, only allowed once at the + * beginning or end of the format string). Note that 'S' allows '-' but 'MI' does not. 'PR': + * Only allowed at the end of the format string; specifies that 'expr' indicates a negative + * number with wrapping angled brackets. * - * @param columnName - * column to check if it is aggregated. A column of any type. - * @group agg_funcs - * @since 2.0.0 + * @param e + * The input string to convert to a number. A column that evaluates to a string. + * @param format + * The format to use to convert the value. A column that evaluates to a string. Must be a + * constant. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a byte. + * Returns a column that evaluates to a decimal. */ - def grouping(columnName: String): Column = grouping(Column(columnName)) + def to_number(e: Column, format: Column): Column = Column.fn("to_number", e, format) /** - * Aggregate function: returns the level of grouping, equals to - * - * {{{ - * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) - * }}} + * Replaces all occurrences of `search` with `replace`. * - * @param cols - * columns to check for. Columns of any type. - * @note - * The list of columns should match with grouping columns exactly, or empty (means all the - * grouping columns). + * @param src + * A column of strings to be replaced. A column that evaluates to a string. + * @param search + * A column of strings. If `search` is not found in `str`, `str` is returned unchanged. A + * column that evaluates to a string. + * @param replace + * A column of strings. If `replace` is not specified or is an empty string, nothing replaces + * the string that is removed from `str`. A column that evaluates to a string. * - * @group agg_funcs - * @since 2.0.0 + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def grouping_id(cols: Column*): Column = Column.fn("grouping_id", cols: _*) + def replace(src: Column, search: Column, replace: Column): Column = + Column.fn("replace", src, search, replace) /** - * Aggregate function: returns the level of grouping, equals to - * - * {{{ - * (grouping(c1) <<; (n-1)) + (grouping(c2) <<; (n-2)) + ... + grouping(cn) - * }}} - * - * @param colName - * the name of the first grouping column. A column of any type. - * @param colNames - * the names of the remaining grouping columns. Columns of any type. - * @note - * The list of columns should match with grouping columns exactly. + * Replaces all occurrences of `search` with `replace`. * - * @group agg_funcs - * @since 2.0.0 - * @return - * Returns a column that evaluates to a long. - */ - @scala.annotation.varargs - def grouping_id(colName: String, colNames: String*): Column = { - grouping_id((Seq(colName) ++ colNames).map(n => Column(n)): _*) - } - - /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with lgConfigK arg. + * @param src + * A column of strings to be replaced. A column that evaluates to a string. + * @param search + * A column of strings. If `search` is not found in `src`, `src` is returned unchanged. A + * column that evaluates to a string. * - * @param e - * the column to compute the sketch on. A column that evaluates to an integral, a string or a - * binary. - * @param lgConfigK - * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column - * that evaluates to an integral. Must be a constant. - * @group agg_funcs + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_sketch_agg(e: Column, lgConfigK: Column): Column = - Column.fn("hll_sketch_agg", e, lgConfigK) + def replace(src: Column, search: Column): Column = Column.fn("replace", src, search) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with lgConfigK arg. + * Splits `str` by delimiter and return requested part of the split (1-based). If any input is + * null, returns null. if `partNum` is out of range of split parts, returns empty string. If + * `partNum` is 0, throws an error. If `partNum` is negative, the parts are counted backward + * from the end of the string. If the `delimiter` is an empty string, the `str` is not split. * - * @param e - * the column to compute the sketch on. A column that evaluates to an integral, a string or a - * binary. - * @param lgConfigK - * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column - * that evaluates to an integral. Must be a constant. - * @group agg_funcs + * @param str + * A column of strings to be split. A column that evaluates to a string. + * @param delimiter + * The delimiter used for split. A column that evaluates to a string. + * @param partNum + * The requested part of the split (1-based). A column that evaluates to an integral. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_sketch_agg(e: Column, lgConfigK: Int): Column = - Column.fn("hll_sketch_agg", e, lit(lgConfigK)) + def split_part(str: Column, delimiter: Column, partNum: Column): Column = + Column.fn("split_part", str, delimiter, partNum) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with lgConfigK arg. + * Returns the substring of `str` that starts at `pos` and is of length `len`, or the slice of + * byte array that starts at `pos` and is of length `len`. * - * @param columnName - * the name of the column to compute the sketch on. A column that evaluates to an integral, a - * string or a binary. - * @param lgConfigK - * the log-base-2 of K, where K is the number of buckets or slots for the HllSketch. A column - * that evaluates to an integral. Must be a constant. - * @group agg_funcs + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. + * @param len + * The length of the substring. A column that evaluates to an integral. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def hll_sketch_agg(columnName: String, lgConfigK: Int): Column = { - hll_sketch_agg(Column(columnName), lgConfigK) - } + def substr(str: Column, pos: Column, len: Column): Column = + Column.fn("substr", str, pos, len) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with default lgConfigK value. + * Returns the substring of `str` that starts at `pos`, or the slice of byte array that starts + * at `pos`. * - * @param e - * the column to compute the sketch on. A column that evaluates to an integral, a string or a - * binary. - * @group agg_funcs + * @param str + * The input from which to take the substring. A column that evaluates to a string or binary. + * @param pos + * The starting position of the substring. A column that evaluates to an integral. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def hll_sketch_agg(e: Column): Column = - Column.fn("hll_sketch_agg", e) + def substr(str: Column, pos: Column): Column = Column.fn("substr", str, pos) /** - * Aggregate function: returns the updatable binary representation of the Datasketches HllSketch - * configured with default lgConfigK value. + * Extracts a part from a URL. * - * @param columnName - * the name of the column to compute the sketch on. A column that evaluates to an integral, a - * string or a binary. - * @group agg_funcs - * @since 3.5.0 + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. + * @group url_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_sketch_agg(columnName: String): Column = { - hll_sketch_agg(Column(columnName)) - } + def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = + Column.fn("try_parse_url", url, partToExtract, key) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values - * and allowDifferentLgConfigK is set to false. + * Extracts a part from a URL. * - * @param e - * the column containing the HllSketch instances to merge. A column that evaluates to a - * binary. - * @param allowDifferentLgConfigK - * allow sketches with different lgConfigK values to be merged. A column that evaluates to a - * boolean. Must be a constant. - * @group agg_funcs - * @since 3.5.0 + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @group url_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_union_agg(e: Column, allowDifferentLgConfigK: Column): Column = - Column.fn("hll_union_agg", e, allowDifferentLgConfigK) + def try_parse_url(url: Column, partToExtract: Column): Column = + Column.fn("try_parse_url", url, partToExtract) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values - * and allowDifferentLgConfigK is set to false. + * Extracts a part from a URL. * - * @param e - * the column containing the HllSketch instances to merge. A column that evaluates to a - * binary. - * @param allowDifferentLgConfigK - * allow sketches with different lgConfigK values to be merged. A column that evaluates to a - * boolean. Must be a constant. - * @group agg_funcs + * @param url + * A column of strings, each representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @param key + * The key of a query parameter in the URL. A column that evaluates to a string. + * @group url_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_union_agg(e: Column, allowDifferentLgConfigK: Boolean): Column = - Column.fn("hll_union_agg", e, lit(allowDifferentLgConfigK)) + def parse_url(url: Column, partToExtract: Column, key: Column): Column = + Column.fn("parse_url", url, partToExtract, key) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values - * and allowDifferentLgConfigK is set to false. + * Extracts a part from a URL. * - * @param columnName - * the name of the column containing the HllSketch instances to merge. A column that evaluates - * to a binary. - * @param allowDifferentLgConfigK - * allow sketches with different lgConfigK values to be merged. A column that evaluates to a - * boolean. Must be a constant. - * @group agg_funcs + * @param url + * A column representing a URL. A column that evaluates to a string. + * @param partToExtract + * The part to extract from the URL. A column that evaluates to a string. + * @group url_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_union_agg(columnName: String, allowDifferentLgConfigK: Boolean): Column = { - hll_union_agg(Column(columnName), allowDifferentLgConfigK) - } + def parse_url(url: Column, partToExtract: Column): Column = + Column.fn("parse_url", url, partToExtract) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. + * Formats the arguments in printf-style and returns the result as a string column. * - * @param e - * the column containing the HllSketch instances to merge. A column that evaluates to a - * binary. - * @group agg_funcs + * @param format + * A format string that can contain embedded format tags. A column that evaluates to a string. + * @param arguments + * The values to be used in formatting. Columns that evaluate to any type. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_union_agg(e: Column): Column = - Column.fn("hll_union_agg", e) + @scala.annotation.varargs + def printf(format: Column, arguments: Column*): Column = + Column.fn("printf", (format +: arguments): _*) /** - * Aggregate function: returns the updatable binary representation of the Datasketches - * HllSketch, generated by merging previously created Datasketches HllSketch instances via a - * Datasketches Union instance. Throws an exception if sketches have different lgConfigK values. + * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding + * scheme. * - * @param columnName - * the name of the column containing the HllSketch instances to merge. A column that evaluates - * to a binary. - * @group agg_funcs + * @param str + * A URL-encoded string. A column that evaluates to a string. + * @group url_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_union_agg(columnName: String): Column = { - hll_union_agg(Column(columnName)) - } + def url_decode(str: Column): Column = Column.fn("url_decode", str) /** - * Aggregate function: returns the kurtosis of the values in a group. + * This is a special version of `url_decode` that performs the same operation, but returns a + * NULL value instead of raising an error if the decoding cannot be performed. * - * @param e - * the column to compute the kurtosis on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param str + * A URL-encoded string. A column that evaluates to a string. + * @group url_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def kurtosis(e: Column): Column = Column.fn("kurtosis", e) + def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) /** - * Aggregate function: returns the kurtosis of the values in a group. + * Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding + * scheme. * - * @param columnName - * the name of the column to compute the kurtosis on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param str + * A string to encode. A column that evaluates to a string. + * @group url_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def kurtosis(columnName: String): Column = kurtosis(Column(columnName)) + def url_encode(str: Column): Column = Column.fn("url_encode", str) /** - * Aggregate function: returns the last value in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * the column to take the last value from. A column of any type. - * @param ignoreNulls - * if true, returns the last non-null value; if all values are null, null is returned. A - * column that evaluates to a boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Returns the position of the first occurrence of `substr` in `str` after position `start`. The + * given `start` and return value are 1-based. * - * @group agg_funcs - * @since 2.0.0 + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. + * @param start + * The 1-based position to start the search from. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def last(e: Column, ignoreNulls: Boolean): Column = - Column.fn("last", false, e, lit(ignoreNulls)) + def position(substr: Column, str: Column, start: Column): Column = + Column.fn("position", substr, str, start) /** - * Aggregate function: returns the last value of the column in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param columnName - * the name of the column to take the last value from. A column of any type. - * @param ignoreNulls - * if true, returns the last non-null value; if all values are null, null is returned. A - * column that evaluates to a boolean. Must be a constant. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Returns the position of the first occurrence of `substr` in `str` after position `1`. The + * return value are 1-based. * - * @group agg_funcs - * @since 2.0.0 + * @param substr + * The substring to search for. A column that evaluates to a string. + * @param str + * The string to search in. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def last(columnName: String, ignoreNulls: Boolean): Column = { - last(Column(columnName), ignoreNulls) - } + def position(substr: Column, str: Column): Column = + Column.fn("position", substr, str) /** - * Aggregate function: returns the last value in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * column to fetch the last value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Returns a boolean. The value is True if str ends with suffix. Returns NULL if either input + * expression is NULL. Otherwise, returns False. Both str or suffix must be of STRING or BINARY + * type. * - * @group agg_funcs - * @since 1.3.0 + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param suffix + * The suffix to test for. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def last(e: Column): Column = last(e, ignoreNulls = false) + def endswith(str: Column, suffix: Column): Column = + Column.fn("endswith", str, suffix) /** - * Aggregate function: returns the last value of the column in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. + * Returns a boolean. The value is True if str starts with prefix. Returns NULL if either input + * expression is NULL. Otherwise, returns False. Both str or prefix must be of STRING or BINARY + * type. * - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. - * - * @group agg_funcs - * @since 1.3.0 + * @param str + * The string to test. A column that evaluates to a string or binary. + * @param prefix + * The prefix to test for. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def last(columnName: String): Column = last(Column(columnName), ignoreNulls = false) + def startswith(str: Column, prefix: Column): Column = + Column.fn("startswith", str, prefix) /** - * Aggregate function: returns the last value in a group. - * - * @param e - * column to fetch the last value for. A column of any type. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the + * result is equivalent to char(n % 256) * - * @group agg_funcs + * @param n + * The code point value. A column that evaluates to an integral. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def last_value(e: Column): Column = Column.fn("last_value", e) + def char(n: Column): Column = Column.fn("char", n) /** - * Aggregate function: returns the last value in a group. - * - * The function by default returns the last values it sees. It will return the last non-null - * value it sees when ignoreNulls is set to true. If all values are null, then null is returned. - * - * @param e - * column to fetch the last value for. A column of any type. - * @param ignoreNulls - * whether to skip null values. A column that evaluates to a boolean. - * @note - * The function is non-deterministic because its results depends on the order of the rows - * which may be non-deterministic after a shuffle. + * Removes the leading and trailing space characters from `str`. * - * @group agg_funcs + * @param str + * The string to trim. A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def last_value(e: Column, ignoreNulls: Column): Column = - Column.fn("last_value", e, ignoreNulls) + def btrim(str: Column): Column = Column.fn("btrim", str) /** - * Aggregate function: returns the most frequent value in a group. + * Remove the leading and trailing `trim` characters from `str`. * - * @param e - * target column to compute on. A column of any type. - * @group agg_funcs - * @since 3.4.0 + * @param str + * The string to trim. A column that evaluates to a string. + * @param trim + * The trim string characters to trim. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def mode(e: Column): Column = Column.fn("mode", e) + def btrim(str: Column, trim: Column): Column = Column.fn("btrim", str, trim) /** - * Aggregate function: returns the most frequent value in a group. - * - * When multiple values have the same greatest frequency then either any of values is returned - * if deterministic is false or is not defined, or the lowest value is returned if deterministic - * is true. + * This is a special version of `to_binary` that performs the same operation, but returns a NULL + * value instead of raising an error if the conversion cannot be performed. * * @param e - * target column to compute on. A column of any type. - * @param deterministic - * if there are multiple equally-frequent results then return the lowest. A boolean. Must be a + * The string to convert. A column that evaluates to a string. + * @param f + * The format to use for the conversion. A column that evaluates to a string. Must be a * constant. - * @group agg_funcs - * @since 4.0.0 + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def mode(e: Column, deterministic: Boolean): Column = Column.fn("mode", e, lit(deterministic)) + def try_to_binary(e: Column, f: Column): Column = Column.fn("try_to_binary", e, f) /** - * Aggregate function: returns the maximum value of the expression in a group. + * This is a special version of `to_binary` that performs the same operation, but returns a NULL + * value instead of raising an error if the conversion cannot be performed. * * @param e - * the target column on which the maximum value is computed. A column of any type. - * @group agg_funcs - * @since 1.3.0 + * The string to convert. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a binary. */ - def max(e: Column): Column = Column.fn("max", e) + def try_to_binary(e: Column): Column = Column.fn("try_to_binary", e) /** - * Aggregate function: returns the maximum value of the column in a group. + * Convert string `e` to a number based on the string format `format`. Returns NULL if the + * string `e` does not match the expected format. The format follows the same semantics as the + * to_number function. * - * @group agg_funcs - * @since 1.3.0 + * @param e + * The string to convert. A column that evaluates to a string. + * @param format + * The format used to convert the string to a number. A column that evaluates to a string. + * Must be a constant. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a decimal. */ - def max(columnName: String): Column = max(Column(columnName)) + def try_to_number(e: Column, format: Column): Column = Column.fn("try_to_number", e, format) /** - * Aggregate function: returns the value associated with the maximum value of ord. - * - * @param e - * the column representing the values to be returned. A column of any type. - * @param ord - * the column that needs to be maximized. A column of any orderable type. - * @note - * The function is non-deterministic so the output order can be different for those associated - * the same values of `e`. + * Returns the character length of string data or number of bytes of binary data. The length of + * string data includes the trailing spaces. The length of binary data includes binary zeros. * - * @group agg_funcs - * @since 3.3.0 + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def max_by(e: Column, ord: Column): Column = Column.fn("max_by", e, ord) + def char_length(str: Column): Column = Column.fn("char_length", str) /** - * Aggregate function: returns an array of values associated with the top `k` values of `ord`. - * - * The result array contains values in descending order by their associated ordering values. - * Returns null if there are no non-null ordering values. - * - * @param e - * the column representing the values to be returned. A column of any type. - * @param ord - * the column that needs to be maximized. A column of any orderable type. - * @param k - * the number of top values to return. An integer. Must be a constant. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. + * Returns the character length of string data or number of bytes of binary data. The length of + * string data includes the trailing spaces. The length of binary data includes binary zeros. * - * @group agg_funcs - * @since 4.2.0 + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def max_by(e: Column, ord: Column, k: Int): Column = Column.fn("max_by", e, ord, lit(k)) + def character_length(str: Column): Column = Column.fn("character_length", str) /** - * Aggregate function: returns an array of values associated with the top `k` values of `ord`. - * - * The result array contains values in descending order by their associated ordering values. - * Returns null if there are no non-null ordering values. - * - * @param e - * the column representing the values to be returned. A column of any type. - * @param ord - * the column that needs to be maximized. A column of any orderable type. - * @param k - * the number of top values to return. A column that evaluates to an integer. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. + * Returns the ASCII character having the binary equivalent to `n`. If n is larger than 256 the + * result is equivalent to chr(n % 256) * - * @group agg_funcs - * @since 4.2.0 + * @param n + * The code point. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a string. */ - def max_by(e: Column, ord: Column, k: Column): Column = Column.fn("max_by", e, ord, k) + def chr(n: Column): Column = Column.fn("chr", n) /** - * Aggregate function: returns the average of the values in a group. Alias for avg. + * Returns a boolean. The value is True if right is found inside left. Returns NULL if either + * input expression is NULL. Otherwise, returns False. Both left or right must be of STRING or + * BINARY type. * - * @param e - * target column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.4.0 + * @param left + * The input to check, may be NULL. A column that evaluates to a string or binary. + * @param right + * The input to find, may be NULL. A column that evaluates to a string or binary. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a boolean. */ - def mean(e: Column): Column = avg(e) + def contains(left: Column, right: Column): Column = Column.fn("contains", left, right) /** - * Aggregate function: returns the average of the values in a group. Alias for avg. + * Returns the `n`-th input, e.g., returns `input2` when `n` is 2. The function returns NULL if + * the index exceeds the length of the array and `spark.sql.ansi.enabled` is set to false. If + * `spark.sql.ansi.enabled` is set to true, it throws ArrayIndexOutOfBoundsException for invalid + * indices. * - * @group agg_funcs - * @since 1.4.0 + * @param inputs + * The index followed by the inputs to select from. Columns where the first evaluates to an + * integral and the rest evaluate to strings or binaries. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def mean(columnName: String): Column = avg(columnName) + @scala.annotation.varargs + def elt(inputs: Column*): Column = Column.fn("elt", inputs: _*) /** - * Aggregate function: returns the median of the values in a group. + * Returns the index (1-based) of the given string (`str`) in the comma-delimited list + * (`strArray`). Returns 0, if the string was not found or if the given string (`str`) contains + * a comma. * - * @param e - * target column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.4.0 + * @param str + * The given string to be found. A column that evaluates to a string. + * @param strArray + * The comma-delimited list. A column that evaluates to a string. + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an integer. */ - def median(e: Column): Column = Column.fn("median", e) + def find_in_set(str: Column, strArray: Column): Column = Column.fn("find_in_set", str, strArray) /** - * Aggregate function: returns the minimum value of the expression in a group. - * - * @param e - * the target column on which the minimum value is computed. A column of any type. - * @group agg_funcs - * @since 1.3.0 - * @return - * Returns a column of the same type as the input. - */ - def min(e: Column): Column = Column.fn("min", e) - - /** - * Aggregate function: returns the minimum value of the column in a group. + * Returns true if str matches `pattern` with `escapeChar`, null if any arguments are null, + * false otherwise. * - * @param columnName - * the name of the column on which the minimum value is computed. A column of an orderable - * type. - * @group agg_funcs - * @since 1.3.0 + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def min(columnName: String): Column = min(Column(columnName)) + def like(str: Column, pattern: Column, escapeChar: Column): Column = + Column.fn("like", str, pattern, escapeChar) /** - * Aggregate function: returns the value associated with the minimum value of ord. - * - * @param e - * the column representing the values that will be returned. A column of any type. - * @param ord - * the column that needs to be minimized. A column of an orderable type. - * @note - * The function is non-deterministic so the output order can be different for those associated - * the same values of `e`. + * Returns true if str matches `pattern` with `escapeChar`('\'), null if any arguments are null, + * false otherwise. * - * @group agg_funcs - * @since 3.3.0 + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a boolean. */ - def min_by(e: Column, ord: Column): Column = Column.fn("min_by", e, ord) + def like(str: Column, pattern: Column): Column = Column.fn("like", str, pattern) /** - * Aggregate function: returns an array of values associated with the bottom `k` values of - * `ord`. - * - * The result array contains values in ascending order by their associated ordering values. - * Returns null if there are no non-null ordering values. - * - * @param e - * the column representing the values that will be returned. A column of any type. - * @param ord - * the column that needs to be minimized. A column of an orderable type. - * @param k - * the number of bottom values to return. An integer. Must be a constant. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. + * Returns true if str matches `pattern` with `escapeChar` case-insensitively, null if any + * arguments are null, false otherwise. * - * @group agg_funcs - * @since 4.2.0 + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @param escapeChar + * The escape character. A column that evaluates to a string. Must be a constant. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def min_by(e: Column, ord: Column, k: Int): Column = Column.fn("min_by", e, ord, lit(k)) + def ilike(str: Column, pattern: Column, escapeChar: Column): Column = + Column.fn("ilike", str, pattern, escapeChar) /** - * Aggregate function: returns an array of values associated with the bottom `k` values of - * `ord`. - * - * The result array contains values in ascending order by their associated ordering values. - * Returns null if there are no non-null ordering values. - * - * @param e - * the column representing the values that will be returned. A column of any type. - * @param ord - * the column that needs to be minimized. A column of an orderable type. - * @param k - * the number of bottom values to return. A column that evaluates to an integral. Must be a - * constant. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle when there are ties in the - * ordering expression. - * @note - * The maximum value of `k` is 100000. + * Returns true if str matches `pattern` with `escapeChar`('\') case-insensitively, null if any + * arguments are null, false otherwise. * - * @group agg_funcs - * @since 4.2.0 + * @param str + * A column that evaluates to a string. + * @param pattern + * The pattern to match. A column that evaluates to a string. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to a boolean. */ - def min_by(e: Column, ord: Column, k: Column): Column = Column.fn("min_by", e, ord, k) + def ilike(str: Column, pattern: Column): Column = Column.fn("ilike", str, pattern) /** - * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given - * percentage(s) with value range in [0.0, 1.0]. + * Returns `str` with all characters changed to lowercase. * - * @param e - * the column to compute the percentile on. A column that evaluates to a numeric or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @group agg_funcs + * @param str + * A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def percentile(e: Column, percentage: Column): Column = Column.fn("percentile", e, percentage) + def lcase(str: Column): Column = Column.fn("lcase", str) /** - * Aggregate function: returns the exact percentile(s) of numeric column `expr` at the given - * percentage(s) with value range in [0.0, 1.0]. + * Returns `str` with all characters changed to uppercase. * - * @param e - * the column to compute the percentile on. A column that evaluates to a numeric or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @param frequency - * the positive frequency with which to weight each value. A column that evaluates to an - * integral. - * @group agg_funcs + * @param str + * A column that evaluates to a string. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def percentile(e: Column, percentage: Column, frequency: Column): Column = - Column.fn("percentile", e, percentage, frequency) + def ucase(str: Column): Column = Column.fn("ucase", str) /** - * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is - * the smallest value in the ordered `col` values (sorted from least to greatest) such that no - * more than `percentage` of `col` values is less than the value or equal to that value. - * - * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating - * point value, it must be between 0.0 and 1.0. - * - * The accuracy parameter is a positive numeric literal which controls approximation accuracy at - * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the - * relative error of the approximation. + * Returns the leftmost `len`(`len` can be string type) characters from the string `str`, if + * `len` is less or equal than 0 the result is an empty string. * - * @param e - * the column to compute the approximate percentile on. A column that evaluates to a numeric - * or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @param accuracy - * a positive numeric literal that controls approximation accuracy at the cost of memory. A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.1.0 + * @param str + * Input column or strings. A column that evaluates to a string or binary. + * @param len + * The number of leftmost characters. A column that evaluates to an integral. + * @group string_funcs + * @since 3.5.0 * @return * Returns a column of the same type as the input. */ - def percentile_approx(e: Column, percentage: Column, accuracy: Column): Column = - Column.fn("percentile_approx", e, percentage, accuracy) + def left(str: Column, len: Column): Column = Column.fn("left", str, len) /** - * Aggregate function: returns the approximate `percentile` of the numeric column `col` which is - * the smallest value in the ordered `col` values (sorted from least to greatest) such that no - * more than `percentage` of `col` values is less than the value or equal to that value. - * - * If percentage is an array, each value must be between 0.0 and 1.0. If it is a single floating - * point value, it must be between 0.0 and 1.0. - * - * The accuracy parameter is a positive numeric literal which controls approximation accuracy at - * the cost of memory. Higher value of accuracy yields better accuracy, 1.0/accuracy is the - * relative error of the approximation. + * Returns the rightmost `len`(`len` can be string type) characters from the string `str`, if + * `len` is less or equal than 0 the result is an empty string. * - * @param e - * the column to compute the approximate percentile on. A column that evaluates to a numeric - * or interval. - * @param percentage - * the percentage in decimal, between 0.0 and 1.0. A column that evaluates to a numeric or an - * array. Must be a constant. - * @param accuracy - * a positive numeric literal that controls approximation accuracy at the cost of memory. A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs + * @param str + * Input column or strings. A column that evaluates to a string. + * @param len + * The number of rightmost characters. A column that evaluates to an integral. + * @group string_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def approx_percentile(e: Column, percentage: Column, accuracy: Column): Column = { - Column.fn("approx_percentile", e, percentage, accuracy) - } + def right(str: Column, len: Column): Column = Column.fn("right", str, len) /** - * Aggregate function: returns the product of all numerical elements in a group. + * Returns `str` enclosed by single quotes and each instance of single quote in it is preceded + * by a backslash. * - * @param e - * the column to compute the product on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.2.0 + * @param str + * A column that evaluates to a string. + * @group string_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def product(e: Column): Column = Column.internalFn("product", e) + def quote(str: Column): Column = Column.fn("quote", str) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Datasketch functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Aggregate function: returns the skewness of the values in a group. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches HllSketch. * - * @param e - * the column to compute the skewness on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param c + * The binary representation of a Datasketches HllSketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long. */ - def skewness(e: Column): Column = Column.fn("skewness", e) + def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) /** - * Aggregate function: returns the skewness of the values in a group. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches HllSketch. * * @param columnName - * the name of the column to compute the skewness on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * Name of the column containing the binary representation of a Datasketches HllSketch. A + * column that evaluates to a binary. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long. */ - def skewness(columnName: String): Column = skewness(Column(columnName)) + def hll_sketch_estimate(columnName: String): Column = { + hll_sketch_estimate(Column(columnName)) + } /** - * Aggregate function: alias for `stddev_samp`. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values. * - * @param e - * the column to compute the standard deviation on. A column that evaluates to a numeric. - * @group agg_funcs + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def std(e: Column): Column = Column.fn("std", e) + def hll_union(c1: Column, c2: Column): Column = + Column.fn("hll_union", c1, c2) /** - * Aggregate function: alias for `stddev_samp`. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values. * - * @param e - * the column to compute the standard deviation on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def stddev(e: Column): Column = Column.fn("stddev", e) + def hll_union(columnName1: String, columnName2: String): Column = { + hll_union(Column(columnName1), Column(columnName2)) + } /** - * Aggregate function: alias for `stddev_samp`. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values and + * allowDifferentLgConfigK is set to false. * - * @param columnName - * the name of the column to compute the standard deviation on. A column that evaluates to a - * numeric. - * @group agg_funcs - * @since 1.6.0 + * @param c1 + * The first binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches HllSketch. A column that evaluates to a + * binary. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def stddev(columnName: String): Column = stddev(Column(columnName)) + def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = + Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) /** - * Aggregate function: returns the sample standard deviation of the expression in a group. + * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches + * Union object. Throws an exception if sketches have different lgConfigK values and + * allowDifferentLgConfigK is set to false. * - * @param e - * the column to compute the sample standard deviation on. A column that evaluates to a - * numeric. - * @group agg_funcs - * @since 1.6.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches HllSketch. + * A column that evaluates to a binary. + * @param allowDifferentLgConfigK + * Allow sketches with different lgConfigK values to be merged (defaults to false). A column + * that evaluates to a boolean. Must be a constant. + * @group sketch_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def stddev_samp(e: Column): Column = Column.fn("stddev_samp", e) + def hll_union( + columnName1: String, + columnName2: String, + allowDifferentLgConfigK: Boolean): Column = { + hll_union(Column(columnName1), Column(columnName2), allowDifferentLgConfigK) + } /** - * Aggregate function: returns the sample standard deviation of the expression in a group. + * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches AnotB object * - * @param columnName - * Name of the column to compute the sample standard deviation on. A column that evaluates to - * a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def stddev_samp(columnName: String): Column = stddev_samp(Column(columnName)) + def theta_difference(c1: Column, c2: Column): Column = + Column.fn("theta_difference", c1, c2) /** - * Aggregate function: returns the population standard deviation of the expression in a group. + * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches AnotB object * - * @param e - * The column to compute the population standard deviation on. A column that evaluates to a - * numeric. - * @group agg_funcs - * @since 1.6.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def stddev_pop(e: Column): Column = Column.fn("stddev_pop", e) + def theta_difference(columnName1: String, columnName2: String): Column = { + theta_difference(Column(columnName1), Column(columnName2)) + } /** - * Aggregate function: returns the population standard deviation of the expression in a group. + * Intersects two binary representations of Datasketches ThetaSketch objects in the input + * columns using a Datasketches Intersection object * - * @param columnName - * Name of the column to compute the population standard deviation on. A column that evaluates - * to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def stddev_pop(columnName: String): Column = stddev_pop(Column(columnName)) + def theta_intersection(c1: Column, c2: Column): Column = + Column.fn("theta_intersection", c1, c2) /** - * Aggregate function: returns the sum of all values in the expression. + * Intersects two binary representations of Datasketches ThetaSketch objects in the input + * columns using a Datasketches Intersection object * - * @param e - * The column to sum. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 1.3.0 + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a binary. */ - def sum(e: Column): Column = Column.fn("sum", e) + def theta_intersection(columnName1: String, columnName2: String): Column = { + theta_intersection(Column(columnName1), Column(columnName2)) + } /** - * Aggregate function: returns the sum of all values in the given column. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches ThetaSketch. * - * @param columnName - * Name of the column to sum. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 1.3.0 + * @param c + * The binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a long. */ - def sum(columnName: String): Column = sum(Column(columnName)) + def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) /** - * Aggregate function: returns the sum of distinct values in the expression. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches ThetaSketch. * - * @group agg_funcs - * @since 1.3.0 + * @param columnName + * Name of the column containing the binary representation of a Datasketches ThetaSketch. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a long. */ - @deprecated("Use sum_distinct", "3.2.0") - def sumDistinct(e: Column): Column = sum_distinct(e) + def theta_sketch_estimate(columnName: String): Column = { + theta_sketch_estimate(Column(columnName)) + } /** - * Aggregate function: returns the sum of distinct values in the expression. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It is configured with the default value of 12 for + * `lgNomEntries`. * - * @group agg_funcs - * @since 1.3.0 + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a + * binary. + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a numeric or interval. + * Returns a column that evaluates to a binary. */ - @deprecated("Use sum_distinct", "3.2.0") - def sumDistinct(columnName: String): Column = sum_distinct(Column(columnName)) + def theta_union(c1: Column, c2: Column): Column = + Column.fn("theta_union", c1, c2) /** - * Aggregate function: returns the sum of distinct values in the expression. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It is configured with the default value of 12 for + * `lgNomEntries`. * - * @param e - * The column to sum distinct values of. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 3.2.0 - * @return - * Returns a column that evaluates to a numeric or interval. + * @param columnName1 + * Name of the column containing the first binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @param columnName2 + * Name of the column containing the second binary representation of a Datasketches + * ThetaSketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a binary. */ - def sum_distinct(e: Column): Column = Column.fn("sum", isDistinct = true, e) + def theta_union(columnName1: String, columnName2: String): Column = { + theta_union(Column(columnName1), Column(columnName2)) + } /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input - * column via a Datasketches Intersection instance. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param e - * The column of Datasketches ThetaSketch instances to intersect. A column that evaluates to a + * @param c1 + * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a * binary. - * @group agg_funcs + * @param c2 + * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to + * a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, + * defaults to 12). A column that evaluates to an integral. Must be a constant. + * @group sketch_funcs * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def theta_intersection_agg(e: Column): Column = - Column.fn("theta_intersection_agg", e) + def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("theta_union", c1, c2, lit(lgNomEntries)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by intersecting the Datasketches ThetaSketch instances in the input - * volumn via a Datasketches Intersection instance. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param columnName - * Name of the column of Datasketches ThetaSketch instances to intersect. A column that - * evaluates to a binary. - * @group agg_funcs + * @param columnName1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param columnName2 + * The second ThetaSketch column to union. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. Must be a constant. + * @group sketch_funcs * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def theta_intersection_agg(columnName: String): Column = - theta_intersection_agg(Column(columnName)) + def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { + theta_union(Column(columnName1), Column(columnName2), lgNomEntries) + } /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the `lgNomEntries` nominal - * entries. + * Unions two binary representations of Datasketches ThetaSketch objects in the input columns + * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal + * entries for the union buffer. * - * @param e - * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, - * binary or array. + * @param c1 + * The first ThetaSketch column to union. A column that evaluates to a binary. + * @param c2 + * The second ThetaSketch column to union. A column that evaluates to a binary. * @param lgNomEntries - * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and - * 26). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. + * @group sketch_funcs * @since 4.1.0 * @return * Returns a column that evaluates to a binary. */ - def theta_sketch_agg(e: Column, lgNomEntries: Column): Column = - Column.fn("theta_sketch_agg", e, lgNomEntries) + def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = + Column.fn("theta_union", c1, c2, lgNomEntries) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the `lgNomEntries` nominal - * entries. + * Subtracts two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. * - * @param e - * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, - * binary or array. - * @param lgNomEntries - * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and - * 26). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_sketch_agg(e: Column, lgNomEntries: Int): Column = - Column.fn("theta_sketch_agg", e, lit(lgNomEntries)) + def tuple_difference_double(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_double", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the `lgNomEntries` nominal - * entries. + * Subtracts two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. * - * @param columnName - * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, - * string, binary or array. - * @param lgNomEntries - * The log-base-2 of nominal entries, which is the size of the sketch (must be between 4 and - * 26). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_sketch_agg(columnName: String, lgNomEntries: Int): Column = - theta_sketch_agg(Column(columnName), lgNomEntries) + def tuple_difference_double(columnName1: String, columnName2: String): Column = + tuple_difference_double(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the default value of 12 for - * `lgNomEntries`. + * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. * - * @param e - * The column to build the ThetaSketch from. A column that evaluates to a numeric, string, - * binary or array. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_sketch_agg(e: Column): Column = - Column.fn("theta_sketch_agg", e) + def tuple_difference_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_integer", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches ThetaSketch - * built with the values in the input column and configured with the default value of 12 for - * `lgNomEntries`. + * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches AnotB object. Returns elements in the + * first sketch that are not in the second sketch. * - * @param columnName - * Name of the column to build the ThetaSketch from. A column that evaluates to a numeric, - * string, binary or array. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to subtract. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_sketch_agg(columnName: String): Column = - theta_sketch_agg(Column(columnName)) + def tuple_difference_integer(columnName1: String, columnName2: String): Column = + tuple_difference_integer(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). It is configured with the default mode of 'sum'. * - * @param e - * The column containing binary ThetaSketch representations. A column that evaluates to a - * binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_union_agg(e: Column, lgNomEntries: Column): Column = - Column.fn("theta_union_agg", e, lgNomEntries) + def tuple_intersection_double(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_double", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). It is configured with the default mode of 'sum'. * - * @param e - * The column containing binary ThetaSketch representations. A column that evaluates to a - * binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_union_agg(e: Column, lgNomEntries: Int): Column = - Column.fn("theta_union_agg", e, lit(lgNomEntries)) + def tuple_intersection_double(columnName1: String, columnName2: String): Column = + tuple_intersection_double(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param columnName - * The name of the column containing binary ThetaSketch representations. A column that - * evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_union_agg(columnName: String, lgNomEntries: Int): Column = - theta_union_agg(Column(columnName), lgNomEntries) + def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_double", c1, c2, lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It is configured with the default value of 12 for - * `lgNomEntries`. - * - * @param e - * The column containing binary ThetaSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.0 - * @return - * Returns a column that evaluates to a binary. - */ - def theta_union_agg(e: Column): Column = - Column.fn("theta_union_agg", e) - - /** - * Aggregate function: returns the compact binary representation of the Datasketches - * ThetaSketch, generated by the union of Datasketches ThetaSketch instances in the input column - * via a Datasketches Union instance. It is configured with the default value of 12 for - * `lgNomEntries`. + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param columnName - * The name of the column containing binary ThetaSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def theta_union_agg(columnName: String): Column = - theta_union_agg(Column(columnName)) + def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = + tuple_intersection_double(Column(columnName1), Column(columnName2), mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. The mode parameter specifies - * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). + * Intersects two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Intersection object. The mode parameter + * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, + * alwaysone). * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_agg_double(e: Column, mode: Column): Column = - Column.fn("tuple_intersection_agg_double", e, mode) + def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_double", c1, c2, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. The mode parameter specifies - * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). It is configured with the default mode of 'sum'. * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_agg_double(e: Column, mode: String): Column = - Column.fn("tuple_intersection_agg_double", e, lit(mode)) + def tuple_intersection_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_integer", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. The mode parameter specifies - * the aggregation mode for numeric summaries during intersection (sum, min, max, alwaysone). + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). It is configured with the default mode of 'sum'. * - * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_agg_double(columnName: String, mode: String): Column = - tuple_intersection_agg_double(Column(columnName), mode) + def tuple_intersection_integer(columnName1: String, columnName2: String): Column = + tuple_intersection_integer(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. It is configured with the - * default mode of 'sum'. + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs + * @param c1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_agg_double(e: Column): Column = - Column.fn("tuple_intersection_agg_double", e) + def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by intersecting the Datasketches TupleSketch instances - * in the input column via a Datasketches Intersection instance. It is configured with the - * default mode of 'sum'. + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). * - * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs + * @param columnName1 + * The first TupleSketch column to intersect. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column to intersect. A column that evaluates to a binary. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_agg_double(columnName: String): Column = - tuple_intersection_agg_double(Column(columnName)) + def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = + tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Intersects two binary representations of Datasketches TupleSketch objects with integer + * summary data type in the input columns using a Datasketches Intersection object. The mode + * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, + * max, alwaysone). * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a - * binary. + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @param mode * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to * a string. Must be a constant. - * @group agg_funcs + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_intersection_agg_integer(e: Column, mode: Column): Column = - Column.fn("tuple_intersection_agg_integer", e, mode) + def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_integer", c1, c2, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with double summary data type. * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(e: Column, mode: String): Column = - Column.fn("tuple_intersection_agg_integer", e, lit(mode)) + def tuple_sketch_estimate_double(c: Column): Column = + Column.fn("tuple_sketch_estimate_double", c) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with double summary data type. * * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group agg_funcs + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(columnName: String, mode: String): Column = - tuple_intersection_agg_integer(Column(columnName), mode) + def tuple_sketch_estimate_double(columnName: String): Column = + tuple_sketch_estimate_double(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. It is configured with - * the default mode of 'sum'. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with integer summary data type. * - * @param e - * The column containing binary TupleSketch representations. A column that evaluates to a + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a * binary. - * @group agg_funcs + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(e: Column): Column = - Column.fn("tuple_intersection_agg_integer", e) + def tuple_sketch_estimate_integer(c: Column): Column = + Column.fn("tuple_sketch_estimate_integer", c) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by intersecting the Datasketches TupleSketch - * instances in the input column via a Datasketches Intersection instance. It is configured with - * the default mode of 'sum'. + * Returns the estimated number of unique values given the binary representation of a + * Datasketches TupleSketch with integer summary data type. * * @param columnName - * The name of the column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a binary. + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. */ - def tuple_intersection_agg_integer(columnName: String): Column = - tuple_intersection_agg_integer(Column(columnName)) + def tuple_sketch_estimate_integer(columnName: String): Column = + tuple_sketch_estimate_integer(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - key: Column, - summary: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_sketch_agg_double", key, summary, lgNomEntries, mode) + def tuple_sketch_summary_double(c: Column): Column = + Column.fn("tuple_sketch_summary_double", c) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - key: Column, - summary: Column, - lgNomEntries: Int, - mode: String): Column = - Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries), lit(mode)) + def tuple_sketch_summary_double(columnName: String): Column = + tuple_sketch_summary_double(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to a numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int, - mode: String): Column = - tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) + def tuple_sketch_summary_double(c: Column, mode: String): Column = + Column.fn("tuple_sketch_summary_double", c, lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double(key: Column, summary: Column, lgNomEntries: Int): Column = - Column.fn("tuple_sketch_agg_double", key, summary, lit(lgNomEntries)) + def tuple_sketch_summary_double(columnName: String, mode: String): Column = + tuple_sketch_summary_double(Column(columnName), mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to a numeric. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_double( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int): Column = - tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) + def tuple_sketch_summary_double(c: Column, mode: Column): Column = + Column.fn("tuple_sketch_summary_double", c, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to a - * numeric. - * @group agg_funcs + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def tuple_sketch_agg_double(key: Column, summary: Column): Column = - Column.fn("tuple_sketch_agg_double", key, summary) + def tuple_sketch_summary_integer(c: Column): Column = + Column.fn("tuple_sketch_summary_integer", c) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is + * configured with the default mode of 'sum'. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to a numeric. - * @group agg_funcs + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def tuple_sketch_agg_double(keyColumnName: String, summaryColumnName: String): Column = - tuple_sketch_agg_double(Column(keyColumnName), Column(summaryColumnName)) + def tuple_sketch_summary_integer(columnName: String): Column = + tuple_sketch_summary_integer(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def tuple_sketch_agg_integer( - key: Column, - summary: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_sketch_agg_integer", key, summary, lgNomEntries, mode) + def tuple_sketch_summary_integer(c: Column, mode: String): Column = + Column.fn("tuple_sketch_summary_integer", c, lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def tuple_sketch_agg_integer( - key: Column, - summary: Column, - lgNomEntries: Int, - mode: String): Column = - Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries), lit(mode)) + def tuple_sketch_summary_integer(columnName: String, mode: String): Column = + tuple_sketch_summary_integer(Column(columnName), mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries and aggregation mode. The mode parameter - * specifies the aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. + * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to an integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def tuple_sketch_agg_integer( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int, - mode: String): Column = - tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries, mode) + def tuple_sketch_summary_integer(c: Column, mode: Column): Column = + Column.fn("tuple_sketch_summary_integer", c, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_integer(key: Column, summary: Column, lgNomEntries: Int): Column = - Column.fn("tuple_sketch_agg_integer", key, summary, lit(lgNomEntries)) + def tuple_sketch_theta_double(c: Column): Column = + Column.fn("tuple_sketch_theta_double", c) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns and - * configured with the `lgNomEntries` nominal entries. It uses the default mode of 'sum'. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to an integral. - * @param lgNomEntries - * the log-base-2 of nominal entries (must be between 4 and 26). A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_integer( - keyColumnName: String, - summaryColumnName: String, - lgNomEntries: Int): Column = - tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName), lgNomEntries) + def tuple_sketch_theta_double(columnName: String): Column = + tuple_sketch_theta_double(Column(columnName)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param key - * the key values against which unique counting occurs. A column that evaluates to an array, a - * binary, a numeric, or a string. - * @param summary - * the summary values against which mode aggregations occur. A column that evaluates to an - * integral. - * @group agg_funcs + * @param c + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a double. */ - def tuple_sketch_agg_integer(key: Column, summary: Column): Column = - Column.fn("tuple_sketch_agg_integer", key, summary) + def tuple_sketch_theta_integer(c: Column): Column = + Column.fn("tuple_sketch_theta_integer", c) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary built with the key and summary values in the input columns. It - * uses the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary + * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 + * and 1.0. * - * @param keyColumnName - * the name of the column containing the key values against which unique counting occurs. A - * column that evaluates to an array, a binary, a numeric, or a string. - * @param summaryColumnName - * the name of the column containing the summary values against which mode aggregations occur. - * A column that evaluates to an integral. - * @group agg_funcs + * @param columnName + * The column containing a binary TupleSketch representation. A column that evaluates to a + * binary. + * @group sketch_funcs + * @since 4.2.0 + * @return + * Returns a column that evaluates to a double. + */ + def tuple_sketch_theta_integer(columnName: String): Column = + tuple_sketch_theta_integer(Column(columnName)) + + /** + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. + * + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_sketch_agg_integer(keyColumnName: String, summaryColumnName: String): Column = - tuple_sketch_agg_integer(Column(keyColumnName), Column(summaryColumnName)) + def tuple_union_double(c1: Column, c2: Column): Column = + Column.fn("tuple_union_double", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * the column containing binary TupleSketch representations to union. A column that evaluates - * to a binary. - * @param lgNomEntries - * the log-base-2 of nominal entries for the union buffer (must be between 4 and 26). A column - * that evaluates to an integral. Must be a constant. - * @param mode - * the summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group agg_funcs + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_double(e: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_agg_double", e, lgNomEntries, mode) + def tuple_union_double(columnName1: String, columnName2: String): Column = + tuple_union_double(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_double(e: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_agg_double", e, lit(lgNomEntries), lit(mode)) + def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_double(columnName: String, lgNomEntries: Int, mode: String): Column = - tuple_union_agg_double(Column(columnName), lgNomEntries, mode) - + def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = + tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) + /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @group agg_funcs + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_double(e: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_agg_double", e, lit(lgNomEntries)) + def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @group agg_funcs + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_double(columnName: String, lgNomEntries: Int): Column = - tuple_union_agg_double(Column(columnName), lgNomEntries) + def tuple_union_double( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. + * Unions two binary representations of Datasketches TupleSketch objects with double summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @group agg_funcs + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an + * integral. + * @param mode + * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that + * evaluates to a string. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_double(e: Column): Column = - Column.fn("tuple_union_agg_double", e) + def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with a double type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_double(columnName: String): Column = - tuple_union_agg_double(Column(columnName)) + def tuple_union_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_union_integer", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It is configured with the + * default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_integer(e: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_agg_integer", e, lgNomEntries, mode) + def tuple_union_integer(columnName1: String, columnName2: String): Column = + tuple_union_integer(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_integer(e: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries), lit(mode)) + def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for numeric - * summaries (sum, min, max, alwaysone). + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of + * 'sum'. * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an * integral. Must be a constant. - * @param mode - * The summary mode: one of "sum", "min", "max", or "alwaysone". A column that evaluates to a - * string. Must be a constant. - * @group agg_funcs + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_integer(columnName: String, lgNomEntries: Int, mode: String): Column = - tuple_union_agg_integer(Column(columnName), lgNomEntries, mode) + def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = + tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_integer(e: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_agg_integer", e, lit(lgNomEntries)) + def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It allows the configuration of - * `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of 'sum'. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. + * @param columnName1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The second TupleSketch column. A column that evaluates to a binary. * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group agg_funcs + * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a + * constant. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_integer(columnName: String, lgNomEntries: Int): Column = - tuple_union_agg_integer(Column(columnName), lgNomEntries) + def tuple_union_integer( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. + * Unions two binary representations of Datasketches TupleSketch objects with integer summary + * data type in the input columns using a Datasketches Union object. It allows the configuration + * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for + * numeric summaries (sum, min, max, alwaysone). * - * @param e - * The input column containing binary TupleSketch representations. A column that evaluates to - * a binary. - * @group agg_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a binary. - */ - def tuple_union_agg_integer(e: Column): Column = - Column.fn("tuple_union_agg_integer", e) - - /** - * Aggregate function: returns the compact binary representation of the Datasketches TupleSketch - * with an integer type summary, generated by the union of Datasketches TupleSketch instances in - * the input column via a Datasketches Union instance. It is configured with the default values - * of 12 for `lgNomEntries` and 'sum' for mode. - * - * @param columnName - * The name of the input column containing binary TupleSketch representations. A column that - * evaluates to a binary. - * @group agg_funcs + * @param c1 + * The first TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The second TupleSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries. A column that evaluates to an integral. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * @group sketch_funcs * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def tuple_union_agg_integer(columnName: String): Column = - tuple_union_agg_integer(Column(columnName)) + def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = + Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * double summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param e - * The input column containing the values to aggregate. A column that evaluates to an - * integral. - * @param k - * The parameter that controls the size and accuracy of the sketch. A column that evaluates to - * an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_bigint(e: Column, k: Column): Column = - Column.fn("kll_sketch_agg_bigint", e, k) + def tuple_difference_theta_double(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_theta_double", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * double summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param e - * The input column containing the values to aggregate. A column that evaluates to an - * integral. - * @param k - * The parameter that controls the size and accuracy of the sketch. A column that evaluates to - * an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_bigint(e: Column, k: Int): Column = - Column.fn("kll_sketch_agg_bigint", e, lit(k)) + def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = + tuple_difference_theta_double(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * integer summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param columnName - * The column containing bigint values to aggregate. A column that evaluates to an integral. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_bigint(columnName: String, k: Int): Column = - kll_sketch_agg_bigint(Column(columnName), k) + def tuple_difference_theta_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_difference_theta_integer", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column with default k value of 200. + * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with + * integer summary data type in the input columns using a Datasketches AnotB object. Returns + * elements in the TupleSketch that are not in the ThetaSketch. * - * @param e - * The column containing bigint values to aggregate. A column that evaluates to an integral. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_bigint(e: Column): Column = - Column.fn("kll_sketch_agg_bigint", e) + def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = + tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllLongsSketch built with the values in the input column with default k value of 200. + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * - * @param columnName - * The column containing bigint values to aggregate. A column that evaluates to an integral. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_bigint(columnName: String): Column = - kll_sketch_agg_bigint(Column(columnName)) + def tuple_intersection_theta_double(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_theta_double", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * - * @param e - * The column containing float values to aggregate. A column that evaluates to a float. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_float(e: Column, k: Column): Column = - Column.fn("kll_sketch_agg_float", e, k) + def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = + tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param e - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_float(e: Column, k: Int): Column = - Column.fn("kll_sketch_agg_float", e, lit(k)) + def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param columnName - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_float(columnName: String, k: Int): Column = - kll_sketch_agg_float(Column(columnName), k) + def tuple_intersection_theta_double( + columnName1: String, + columnName2: String, + mode: String): Column = + tuple_intersection_theta_double(Column(columnName1), Column(columnName2), mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column with default k value of 200. + * Intersects the binary representation of a Datasketches TupleSketch with double summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param e - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_float(e: Column): Column = - Column.fn("kll_sketch_agg_float", e) + def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_theta_double", c1, c2, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllFloatsSketch built with the values in the input column with default k value of 200. + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * - * @param columnName - * The column containing float values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_float(columnName: String): Column = - kll_sketch_agg_float(Column(columnName)) + def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_intersection_theta_integer", c1, c2) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. * - * @param e - * The column containing double values to aggregate. A column that evaluates to a float or - * double. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The TupleSketch column. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_double(e: Column, k: Column): Column = - Column.fn("kll_sketch_agg_double", e, k) + def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = + tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param e - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. + * Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_double(e: Column, k: Int): Column = - Column.fn("kll_sketch_agg_double", e, lit(k)) + def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = + Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column. The optional k parameter controls - * the size and accuracy of the sketch (default 200, range 8-65535). + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param columnName - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @param k - * The k parameter that controls size and accuracy (default 200, range 8-65535). A column that - * evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.0 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_double(columnName: String, k: Int): Column = - kll_sketch_agg_double(Column(columnName), k) + def tuple_intersection_theta_integer( + columnName1: String, + columnName2: String, + mode: String): Column = + tuple_intersection_theta_integer(Column(columnName1), Column(columnName2), mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column with default k value of 200. + * Intersects the binary representation of a Datasketches TupleSketch with integer summary data + * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection + * object. The mode parameter specifies the aggregation mode for numeric summaries during + * intersection (sum, min, max, alwaysone). * - * @param e - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_double(e: Column): Column = - Column.fn("kll_sketch_agg_double", e) + def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = + Column.fn("tuple_intersection_theta_integer", c1, c2, mode) /** - * Aggregate function: returns the compact binary representation of the Datasketches - * KllDoublesSketch built with the values in the input column with default k value of 200. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param columnName - * The column containing double values to aggregate. A column that evaluates to a numeric. - * @group agg_funcs - * @since 4.1.0 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_sketch_agg_double(columnName: String): Column = - kll_sketch_agg_double(Column(columnName)) + def tuple_union_theta_double(c1: Column, c2: Column): Column = + Column.fn("tuple_union_theta_double", c1, c2) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range - * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * The column containing binary KllLongsSketch representations to merge. A column that - * evaluates to a binary. - * @param k - * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_bigint(e: Column, k: Column): Column = - Column.fn("kll_merge_agg_bigint", e, k) + def tuple_union_theta_double(columnName1: String, columnName2: String): Column = + tuple_union_theta_double(Column(columnName1), Column(columnName2)) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range - * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param e - * The column containing binary KllLongsSketch representations to merge. A column that - * evaluates to a binary. - * @param k - * The k parameter that controls size and accuracy of the merged sketch (range 8-65535). A - * column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_bigint(e: Column, k: Int): Column = - Column.fn("kll_merge_agg_bigint", e, lit(k)) + def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. The optional k parameter controls the size and accuracy of the merged sketch (range - * 8-65535). If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param columnName - * The column containing binary KllLongsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integral. - * @group agg_funcs - * @since 4.1.2 + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_bigint(columnName: String, k: Int): Column = - kll_merge_agg_bigint(Column(columnName), k) + def tuple_union_theta_double( + columnName1: String, + columnName2: String, + lgNomEntries: Int): Column = + tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param e - * The column containing binary KllLongsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_bigint(e: Column): Column = - Column.fn("kll_merge_agg_bigint", e) + def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) /** - * Aggregate function: merges binary KllLongsSketch representations and returns the merged - * sketch. If k is not specified, the merged sketch adopts the k value from the first input - * sketch. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * The column containing binary KllLongsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 - * @return - * Returns a column that evaluates to a binary. - */ - def kll_merge_agg_bigint(columnName: String): Column = - kll_merge_agg_bigint(Column(columnName)) - - /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. - * - * @param e - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integral. - * @group agg_funcs - * @since 4.1.2 - * @return - * Returns a column that evaluates to a binary. - */ - def kll_merge_agg_float(e: Column, k: Column): Column = - Column.fn("kll_merge_agg_float", e, k) - - /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. - * - * @param e - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param columnName1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_float(e: Column, k: Int): Column = - Column.fn("kll_merge_agg_float", e, lit(k)) + def tuple_union_theta_double( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with double summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param c1 + * The TupleSketch column with double summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_float(columnName: String, k: Int): Column = - kll_merge_agg_float(Column(columnName), k) + def tuple_union_theta_double( + c1: Column, + c2: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_union_theta_double", c1, c2, lgNomEntries, mode) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param e - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_float(e: Column): Column = - Column.fn("kll_merge_agg_float", e) + def tuple_union_theta_integer(c1: Column, c2: Column): Column = + Column.fn("tuple_union_theta_integer", c1, c2) /** - * Aggregate function: merges binary KllFloatsSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is + * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. * - * @param columnName - * The column containing binary KllFloatsSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_float(columnName: String): Column = - kll_merge_agg_float(Column(columnName)) + def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = + tuple_union_theta_integer(Column(columnName1), Column(columnName2)) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param e - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_double(e: Column, k: Column): Column = - Column.fn("kll_merge_agg_double", e, k) + def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = + Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses + * the default mode of 'sum'. * - * @param e - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_double(e: Column, k: Int): Column = - Column.fn("kll_merge_agg_double", e, lit(k)) + def tuple_union_theta_integer( + columnName1: String, + columnName2: String, + lgNomEntries: Int): Column = + tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * The optional k parameter controls the size and accuracy of the merged sketch (range 8-65535). - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @param k - * The k parameter that controls size and accuracy (range 8-65535). A column that evaluates to - * an integer. Must be a constant. - * @group agg_funcs - * @since 4.1.2 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_double(columnName: String, k: Int): Column = - kll_merge_agg_double(Column(columnName), k) + def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = + Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param e - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param columnName1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param columnName2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a + * string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_double(e: Column): Column = - Column.fn("kll_merge_agg_double", e) + def tuple_union_theta_integer( + columnName1: String, + columnName2: String, + lgNomEntries: Int, + mode: String): Column = + tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) /** - * Aggregate function: merges binary KllDoublesSketch representations and returns merged sketch. - * If k is not specified, the merged sketch adopts the k value from the first input sketch. + * Unions the binary representation of a Datasketches TupleSketch with integer summary data type + * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It + * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the + * aggregation mode for numeric summaries (sum, min, max, alwaysone). * - * @param columnName - * The column containing binary KllDoublesSketch representations. A column that evaluates to a - * binary. - * @group agg_funcs - * @since 4.1.2 + * @param c1 + * The TupleSketch column with integer summaries. A column that evaluates to a binary. + * @param c2 + * The ThetaSketch column. A column that evaluates to a binary. + * @param lgNomEntries + * The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12). A column that + * evaluates to an integral. Must be a constant. + * @param mode + * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to + * a string. Must be a constant. + * @group sketch_funcs + * @since 4.2.0 * @return * Returns a column that evaluates to a binary. */ - def kll_merge_agg_double(columnName: String): Column = - kll_merge_agg_double(Column(columnName)) - - /** - * Aggregate function: returns the concatenation of non-null input values. - * - * @param e - * The target column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs - * @since 4.0.0 - * @return - * Returns a column of the same type as the input. - */ - def listagg(e: Column): Column = Column.fn("listagg", e) + def tuple_union_theta_integer( + c1: Column, + c2: Column, + lgNomEntries: Column, + mode: Column): Column = + Column.fn("tuple_union_theta_integer", c1, c2, lgNomEntries, mode) /** - * Aggregate function: returns the concatenation of non-null input values, separated by the - * delimiter. + * Returns a string with human readable summary information about the KLL bigint sketch. * * @param e - * The target column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * The delimiter used to separate the values. A column that evaluates to a string or binary. - * Must be a constant. - * @group agg_funcs - * @since 4.0.0 + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def listagg(e: Column, delimiter: Column): Column = Column.fn("listagg", e, delimiter) + def kll_sketch_to_string_bigint(e: Column): Column = + Column.fn("kll_sketch_to_string_bigint", e) /** - * Aggregate function: returns the concatenation of distinct non-null input values. + * Returns a string with human readable summary information about the KLL float sketch. * * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs - * @since 4.0.0 + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a string. */ - def listagg_distinct(e: Column): Column = Column.fn("listagg", isDistinct = true, e) + def kll_sketch_to_string_float(e: Column): Column = + Column.fn("kll_sketch_to_string_float", e) /** - * Aggregate function: returns the concatenation of distinct non-null input values, separated by - * the delimiter. + * Returns a string with human readable summary information about the KLL double sketch. * * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * the delimiter to separate the values. A column that evaluates to a string or binary. Must - * be a constant. - * @group agg_funcs - * @since 4.0.0 + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return * Returns a column that evaluates to a string. */ - def listagg_distinct(e: Column, delimiter: Column): Column = - Column.fn("listagg", isDistinct = true, e, delimiter) + def kll_sketch_to_string_double(e: Column): Column = + Column.fn("kll_sketch_to_string_double", e) /** - * Aggregate function: returns the concatenation of non-null input values. Alias for `listagg`. + * Returns the number of items collected in the KLL bigint sketch. * * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs - * @since 4.0.0 + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def string_agg(e: Column): Column = Column.fn("string_agg", e) + def kll_sketch_get_n_bigint(e: Column): Column = + Column.fn("kll_sketch_get_n_bigint", e) /** - * Aggregate function: returns the concatenation of non-null input values, separated by the - * delimiter. Alias for `listagg`. + * Returns the number of items collected in the KLL float sketch. * * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * the delimiter to separate the values. A column that evaluates to a string or binary. Must - * be a constant. - * @group agg_funcs - * @since 4.0.0 + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a long. */ - def string_agg(e: Column, delimiter: Column): Column = Column.fn("string_agg", e, delimiter) + def kll_sketch_get_n_float(e: Column): Column = + Column.fn("kll_sketch_get_n_float", e) /** - * Aggregate function: returns the concatenation of distinct non-null input values. Alias for - * `listagg`. + * Returns the number of items collected in the KLL double sketch. * * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @group agg_funcs - * @since 4.0.0 + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def string_agg_distinct(e: Column): Column = Column.fn("string_agg", isDistinct = true, e) + def kll_sketch_get_n_double(e: Column): Column = + Column.fn("kll_sketch_get_n_double", e) /** - * Aggregate function: returns the concatenation of distinct non-null input values, separated by - * the delimiter. Alias for `listagg`. + * Merges two KLL bigint sketch buffers together into one. * - * @param e - * the column to compute on. A column that evaluates to a string or binary. - * @param delimiter - * the delimiter to separate the values. A column that evaluates to a string or binary. Must - * be a constant. - * @group agg_funcs - * @since 4.0.0 + * @param left + * The first KLL bigint sketch. A column that evaluates to a binary. + * @param right + * The second KLL bigint sketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a binary. */ - def string_agg_distinct(e: Column, delimiter: Column): Column = - Column.fn("string_agg", isDistinct = true, e, delimiter) + def kll_sketch_merge_bigint(left: Column, right: Column): Column = + Column.fn("kll_sketch_merge_bigint", left, right) /** - * Aggregate function: alias for `var_samp`. + * Merges two KLL float sketch buffers together into one. * - * @param e - * the column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param left + * The first KLL float sketch. A column that evaluates to a binary. + * @param right + * The second KLL float sketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def variance(e: Column): Column = Column.fn("variance", e) + def kll_sketch_merge_float(left: Column, right: Column): Column = + Column.fn("kll_sketch_merge_float", left, right) /** - * Aggregate function: alias for `var_samp`. + * Merges two KLL double sketch buffers together into one. * - * @group agg_funcs - * @since 1.6.0 + * @param left + * The first KLL double sketch. A column that evaluates to a binary. + * @param right + * The second KLL double sketch. A column that evaluates to a binary. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a binary. */ - def variance(columnName: String): Column = variance(Column(columnName)) + def kll_sketch_merge_double(left: Column, right: Column): Column = + Column.fn("kll_sketch_merge_double", left, right) /** - * Aggregate function: returns the unbiased variance of the values in a group. + * Extracts a quantile value from a KLL bigint sketch given an input rank value. The rank can be + * a single value or an array. * - * @param e - * the column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. Must be a constant. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a long, or an array of longs when `rank` is an array. */ - def var_samp(e: Column): Column = Column.fn("var_samp", e) + def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = + Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) /** - * Aggregate function: returns the unbiased variance of the values in a group. + * Extracts a quantile value from a KLL float sketch given an input rank value. The rank can be + * a single value or an array. * - * @group agg_funcs - * @since 1.6.0 + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a float, or an array of floats when `rank` is an array. */ - def var_samp(columnName: String): Column = var_samp(Column(columnName)) + def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = + Column.fn("kll_sketch_get_quantile_float", sketch, rank) /** - * Aggregate function: returns the population variance of the values in a group. + * Extracts a quantile value from a KLL double sketch given an input rank value. The rank can be + * a single value or an array. * - * @param e - * the column to compute on. A column that evaluates to a numeric. - * @group agg_funcs - * @since 1.6.0 + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param rank + * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or + * an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a double, or an array of doubles when `rank` is an + * array. */ - def var_pop(e: Column): Column = Column.fn("var_pop", e) + def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = + Column.fn("kll_sketch_get_quantile_double", sketch, rank) /** - * Aggregate function: returns the population variance of the values in a group. + * Extracts a rank value from a KLL bigint sketch given an input quantile value. The quantile + * can be a single value or an array. * - * @group agg_funcs - * @since 1.6.0 + * @param sketch + * The KLL bigint sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to an integral or an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ - def var_pop(columnName: String): Column = var_pop(Column(columnName)) + def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = + Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) /** - * Aggregate function: returns the average of the independent variable for non-null pairs in a - * group, where `y` is the dependent variable and `x` is the independent variable. + * Extracts a rank value from a KLL float sketch given an input quantile value. The quantile can + * be a single value or an array. * - * @param y - * the dependent variable. A column that evaluates to a numeric. - * @param x - * the independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param sketch + * The KLL float sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to lookup. A column that evaluates to a numeric or an array. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ - def regr_avgx(y: Column, x: Column): Column = Column.fn("regr_avgx", y, x) + def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = + Column.fn("kll_sketch_get_rank_float", sketch, quantile) /** - * Aggregate function: returns the average of the dependent variable for non-null pairs in a - * group, where `y` is the dependent variable and `x` is the independent variable. + * Extracts a rank value from a KLL double sketch given an input quantile value. The quantile + * can be a single value or an array. * - * @param y - * the dependent variable. A column that evaluates to a numeric. - * @param x - * the independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param sketch + * The KLL double sketch binary representation. A column that evaluates to a binary. + * @param quantile + * The quantile value(s) to look up. A column that evaluates to a numeric or an array. Must be + * a constant. + * @group sketch_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an + * array. */ - def regr_avgy(y: Column, x: Column): Column = Column.fn("regr_avgy", y, x) + def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = + Column.fn("kll_sketch_get_rank_double", sketch, quantile) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // DateTime functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Aggregate function: returns the number of non-null number pairs in a group, where `y` is the - * dependent variable and `x` is the independent variable. + * Returns the date that is `numMonths` after `startDate`. * - * @param y - * the dependent variable. A column that evaluates to a numeric. - * @param x - * the independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param startDate + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param numMonths + * The number of months to add to `startDate`, can be negative to subtract months. A column + * that evaluates to an integer. * @return - * Returns a column that evaluates to a long. + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def regr_count(y: Column, x: Column): Column = Column.fn("regr_count", y, x) + def add_months(startDate: Column, numMonths: Int): Column = + add_months(startDate, lit(numMonths)) /** - * Aggregate function: returns the intercept of the univariate linear regression line for - * non-null pairs in a group, where `y` is the dependent variable and `x` is the independent - * variable. + * Returns the date that is `numMonths` after `startDate`. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @param startDate + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param numMonths + * A column of the number of months to add to `startDate`, can be negative to subtract months. + * A column that evaluates to an integer. * @return - * Returns a column that evaluates to a double. + * A date, or null if `startDate` was a string that could not be cast to a date. Returns a + * column that evaluates to a date. + * @group datetime_funcs + * @since 3.0.0 */ - def regr_intercept(y: Column, x: Column): Column = Column.fn("regr_intercept", y, x) + def add_months(startDate: Column, numMonths: Column): Column = + Column.fn("add_months", startDate, numMonths) /** - * Aggregate function: returns the coefficient of determination for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. + * Returns the current date at the start of query evaluation as a date column. All calls of + * current_date within the same query return the same value. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a date. */ - def regr_r2(y: Column, x: Column): Column = Column.fn("regr_r2", y, x) + def curdate(): Column = Column.fn("curdate") /** - * Aggregate function: returns the slope of the linear regression line for non-null pairs in a - * group, where `y` is the dependent variable and `x` is the independent variable. + * Returns the current date at the start of query evaluation as a date column. All calls of + * current_date within the same query return the same value. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @group datetime_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a date. */ - def regr_slope(y: Column, x: Column): Column = Column.fn("regr_slope", y, x) + def current_date(): Column = Column.fn("current_date") /** - * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(x) for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. + * Returns the current session local timezone. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def regr_sxx(y: Column, x: Column): Column = Column.fn("regr_sxx", y, x) + def current_timezone(): Column = Column.fn("current_timezone") /** - * Aggregate function: returns REGR_COUNT(y, x) * COVAR_POP(y, x) for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. + * Returns the current timestamp at the start of query evaluation as a timestamp column. All + * calls of current_timestamp within the same query return the same value. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs - * @since 3.5.0 + * @group datetime_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a timestamp. */ - def regr_sxy(y: Column, x: Column): Column = Column.fn("regr_sxy", y, x) + def current_timestamp(): Column = Column.fn("current_timestamp") /** - * Aggregate function: returns REGR_COUNT(y, x) * VAR_POP(y) for non-null pairs in a group, - * where `y` is the dependent variable and `x` is the independent variable. + * Returns the current timestamp at the start of query evaluation. * - * @param y - * The dependent variable. A column that evaluates to a numeric. - * @param x - * The independent variable. A column that evaluates to a numeric. - * @group agg_funcs + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a timestamp. */ - def regr_syy(y: Column, x: Column): Column = Column.fn("regr_syy", y, x) + def now(): Column = Column.fn("now") /** - * Aggregate function: returns some value of `e` for a group of rows. + * Returns the current timestamp without time zone at the start of query evaluation as a + * timestamp without time zone column. All calls of localtimestamp within the same query return + * the same value. * - * @param e - * The column to return some value from. A column of any type. - * @group agg_funcs - * @since 3.5.0 + * @group datetime_funcs + * @since 3.3.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a timestamp. */ - def any_value(e: Column): Column = Column.fn("any_value", e) + def localtimestamp(): Column = Column.fn("localtimestamp") /** - * Aggregate function: returns some value of `e` for a group of rows. If `ignoreNulls` is true, - * returns only non-null values. + * Converts a date/timestamp/string to a value of string in the format specified by the date + * format given by the second argument. * - * @param e - * The column to return some value from. A column of any type. - * @param ignoreNulls - * If true, returns only non-null values. A column that evaluates to a boolean. Must be a - * constant. - * @group agg_funcs - * @since 3.5.0 + * See Datetime + * Patterns for valid date and time format patterns + * + * @param dateExpr + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp or time. + * @param format + * A pattern `dd.MM.yyyy` would return a string like `18.03.1993`. A column that evaluates to + * a string. * @return - * Returns a column of the same type as the input. + * A string, or null if `dateExpr` was a string that could not be cast to a timestamp. Returns + * a column that evaluates to a string. + * @note + * Use specialized functions like [[year]] whenever possible as they benefit from a + * specialized implementation. + * @throws IllegalArgumentException + * if the `format` pattern is invalid + * @group datetime_funcs + * @since 1.5.0 */ - def any_value(e: Column, ignoreNulls: Column): Column = - Column.fn("any_value", e, ignoreNulls) + def date_format(dateExpr: Column, format: String): Column = + Column.fn("date_format", dateExpr, lit(format)) /** - * Aggregate function: returns the number of `TRUE` values for the expression. + * Returns the date that is `days` days after `start` * - * @param e - * The expression to count TRUE values of. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * The number of days to add to `start`, can be negative to subtract days. A column that + * evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a long. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def count_if(e: Column): Column = Column.fn("count_if", e) + def date_add(start: Column, days: Int): Column = date_add(start, lit(days)) /** - * Aggregate function: computes a histogram on numeric 'expr' using nb bins. The return value is - * an array of (x,y) pairs representing the centers of the histogram's bins. As the value of - * 'nb' is increased, the histogram approximation gets finer-grained, but may yield artifacts - * around outliers. In practice, 20-40 histogram bins appear to work well, with more bins being - * required for skewed or smaller datasets. Note that this function creates a histogram with - * non-uniform bin widths. It offers no guarantees in terms of the mean-squared-error of the - * histogram, but in practice is comparable to the histograms produced by the R/S-Plus - * statistical computing packages. Note: the output type of the 'x' field in the return value is - * propagated from the input value consumed in the aggregate function. + * Returns the date that is `days` days after `start` * - * @param e - * The column to compute the histogram on. A column that evaluates to a numeric. - * @param nBins - * The number of histogram bins. A column that evaluates to an integral. Must be a constant. - * @group agg_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to an array. - */ - def histogram_numeric(e: Column, nBins: Column): Column = - Column.fn("histogram_numeric", e, nBins) - - /** - * Aggregate function: returns true if all values of `e` are true. - * - * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a boolean. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 3.0.0 */ - def every(e: Column): Column = Column.fn("every", e) + def date_add(start: Column, days: Column): Column = Column.fn("date_add", start, days) /** - * Aggregate function: returns true if all values of `e` are true. + * Returns the date that is `days` days after `start` * - * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * A column of the number of days to add to `start`, can be negative to subtract days. A + * column that evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a boolean. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 3.5.0 */ - def bool_and(e: Column): Column = Column.fn("bool_and", e) + def dateadd(start: Column, days: Column): Column = Column.fn("dateadd", start, days) /** - * Aggregate function: returns true if at least one value of `e` is true. + * Returns the date that is `days` days before `start` * - * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * The number of days to subtract from `start`, can be negative to add days. A column that + * evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a boolean. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def some(e: Column): Column = Column.fn("some", e) + def date_sub(start: Column, days: Int): Column = date_sub(start, lit(days)) /** - * Aggregate function: returns true if at least one value of `e` is true. + * Returns the date that is `days` days before `start` * - * @param e - * The expression to evaluate. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param days + * A column of the number of days to subtract from `start`, can be negative to add days. A + * column that evaluates to an integer, short, or byte. * @return - * Returns a column that evaluates to a boolean. + * A date, or null if `start` was a string that could not be cast to a date. Returns a column + * that evaluates to a date. + * @group datetime_funcs + * @since 3.0.0 */ - def any(e: Column): Column = Column.fn("any", e) + def date_sub(start: Column, days: Column): Column = + Column.fn("date_sub", start, days) /** - * Aggregate function: returns true if at least one value of `e` is true. + * Returns the number of days from `start` to `end`. * - * @param e - * column to check if at least one value is true. A column that evaluates to a boolean. - * @group agg_funcs - * @since 3.5.0 + * Only considers the date part of the input. For example: + * {{{ + * datediff("2018-01-10 00:00:00", "2018-01-09 23:59:59") + * // returns 1 + * }}} + * + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * Returns a column that evaluates to a boolean. + * An integer, or null if either `end` or `start` were strings that could not be cast to a + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def bool_or(e: Column): Column = Column.fn("bool_or", e) + def datediff(end: Column, start: Column): Column = Column.fn("datediff", end, start) /** - * Aggregate function: returns the bitwise AND of all non-null input values, or null if none. + * Returns the number of days from `start` to `end`. * - * @param e - * target column to compute on. A column that evaluates to an integral. - * @group agg_funcs - * @since 3.5.0 + * Only considers the date part of the input. For example: + * {{{ + * date_diff("2018-01-10 00:00:00", "2018-01-09 23:59:59") + * // returns 1 + * }}} + * + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * Returns a column of the same type as the input. + * An integer, or null if either `end` or `start` were strings that could not be cast to a + * date. Negative if `end` is before `start`. Returns a column that evaluates to an integer. + * @group datetime_funcs + * @since 3.5.0 */ - def bit_and(e: Column): Column = Column.fn("bit_and", e) + def date_diff(end: Column, start: Column): Column = Column.fn("date_diff", end, start) /** - * Aggregate function: returns the bitwise OR of all non-null input values, or null if none. + * Create date from the number of `days` since 1970-01-01. * - * @param e - * target column to compute on. A column that evaluates to an integral. - * @group agg_funcs + * @param days + * The number of days since 1970-01-01. A column that evaluates to an integral. + * @group datetime_funcs * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a date. */ - def bit_or(e: Column): Column = Column.fn("bit_or", e) + def date_from_unix_date(days: Column): Column = Column.fn("date_from_unix_date", days) /** - * Aggregate function: returns the bitwise XOR of all non-null input values, or null if none. - * + * Extracts the year as an integer from a given date/timestamp/string. * @param e - * target column to compute on. A column that evaluates to an integral. - * @group agg_funcs - * @since 3.5.0 + * The date, timestamp or string to extract the year from. A column that evaluates to a date, + * timestamp or string. * @return - * Returns a column of the same type as the input. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def bit_xor(e: Column): Column = Column.fn("bit_xor", e) + def year(e: Column): Column = Column.fn("year", e) /** - * Returns the mean calculated from values of a group and the result is null on overflow. - * + * Extracts the quarter as an integer from a given date/timestamp/string. * @param e - * the value to compute the mean of. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 3.5.0 + * The date, timestamp or string to extract the quarter from. A column that evaluates to a + * date, timestamp or string. * @return - * Returns a column that evaluates to a double. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def try_avg(e: Column): Column = Column.fn("try_avg", e) + def quarter(e: Column): Column = Column.fn("quarter", e) /** - * Returns the sum calculated from values of a group and the result is null on overflow. - * + * Extracts the month as an integer from a given date/timestamp/string. * @param e - * the value to compute the sum of. A column that evaluates to a numeric or interval. - * @group agg_funcs - * @since 3.5.0 + * The date, timestamp or string to extract the month from. A column that evaluates to a date, + * timestamp or string. * @return - * Returns a column that evaluates to a numeric. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def try_sum(e: Column): Column = Column.fn("try_sum", e) + def month(e: Column): Column = Column.fn("month", e) /** - * Returns a bitmap with the positions of the bits set from all the values from the input - * column. The input column will most likely be bitmap_bit_position(). - * - * @param col - * The input column will most likely be bitmap_bit_position(). A column that evaluates to an - * integral. - * @group agg_funcs - * @since 3.5.0 + * Extracts the day of the week as an integer from a given date/timestamp/string. Ranges from 1 + * for a Sunday through to 7 for a Saturday + * @param e + * The date, timestamp or string to extract the day of the week from. A column that evaluates + * to a date, timestamp or string. * @return - * Returns a column that evaluates to a binary. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 2.3.0 */ - def bitmap_construct_agg(col: Column): Column = - Column.fn("bitmap_construct_agg", col) + def dayofweek(e: Column): Column = Column.fn("dayofweek", e) /** - * Returns a bitmap that is the bitwise OR of all of the bitmaps from the input column. The - * input column should be bitmaps created from bitmap_construct_agg(). - * - * @param col - * The input column should be bitmaps created from bitmap_construct_agg(). A column that - * evaluates to a binary. - * @group agg_funcs - * @since 3.5.0 + * Extracts the day of the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * Returns a column that evaluates to a binary. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def bitmap_or_agg(col: Column): Column = Column.fn("bitmap_or_agg", col) + def dayofmonth(e: Column): Column = Column.fn("dayofmonth", e) /** - * Returns a bitmap that is the bitwise AND of all of the bitmaps from the input column. The - * input column should be bitmaps created from bitmap_construct_agg(). - * - * @param col - * The input column should be bitmaps created from bitmap_construct_agg(). A column that - * evaluates to a binary. - * @group agg_funcs - * @since 4.1.0 + * Extracts the day of the month as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the month from. A column that evaluates + * to a date, timestamp or string. * @return - * Returns a column that evaluates to a binary. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 3.5.0 */ - def bitmap_and_agg(col: Column): Column = Column.fn("bitmap_and_agg", col) + def day(e: Column): Column = Column.fn("day", e) /** - * Returns a bitmap that is the bitwise XOR of all of the bitmaps from the input column. The - * input column should be bitmaps created from bitmap_construct_agg(). - * - * @param col - * A column containing bitmaps created by bitmap_construct_agg() and evaluating to binary - * data. - * @group agg_funcs - * @since 4.4.0 + * Extracts the day of the year as an integer from a given date/timestamp/string. + * @param e + * The date, timestamp or string to extract the day of the year from. A column that evaluates + * to a date, timestamp or string. * @return - * Returns a column that evaluates to a binary. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def bitmap_xor_agg(col: Column): Column = Column.fn("bitmap_xor_agg", col) + def dayofyear(e: Column): Column = Column.fn("dayofyear", e) /** - * Aggregate function: returns a list of objects with duplicates. - * + * Extracts the hours as an integer from a given date/time/timestamp/string. The input may also + * be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in + * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @param e - * the input column. A column that evaluates to any type. - * @note - * The function is non-deterministic because the order of collected results depends on the - * order of the rows which may be non-deterministic after a shuffle. - * @group agg_funcs - * @since 3.5.0 + * The column to extract the hours from. A column that evaluates to a date, time, timestamp or + * string. * @return - * Returns a column that evaluates to an array. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def array_agg(e: Column): Column = Column.fn("array_agg", e) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Window Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def hour(e: Column): Column = Column.fn("hour", e) /** - * Window function: computes the differences between consecutive cumulative counter values in a - * time series, thereby converting the counter from the cumulative to the delta format. - * - * Gracefully handles counter resets by returning NULL. Counter resets are detected when the - * counter value decreases. - * - * Use the PARTITION BY clause of the window to separate independent counters. This is done by - * specifying all columns which uniquely identify a time series. These are typically the counter - * name and any attributes tied to the counter. - * - * Use the ORDER BY clause of the window to order the observations by the associated timestamp - * in ascending order. - * - * @param value - * A cumulative counter. Must be a numeric data type. Must be non-negative. + * Extracts a part of the date/timestamp or interval source. * + * @param field + * selects which part of the source should be extracted. + * @param source + * a date, time, timestamp or interval column from where `field` should be extracted. * @return - * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. Returns a column of the - * same type as the input. - * @group window_funcs - * @since 4.3.0 + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. + * @group datetime_funcs + * @since 3.5.0 */ - def counter_diff(value: Column): Column = Column.fn("counter_diff", value) + def extract(field: Column, source: Column): Column = { + Column.fn("extract", field, source) + } /** - * Window function: computes the differences between consecutive cumulative counter values in a - * time series, thereby converting the counter from the cumulative to the delta format. - * - * Gracefully handles counter resets by returning NULL. Counter resets are detected when the - * counter value decreases, or when the start time advances between rows. - * - * Use the PARTITION BY clause of the window to separate independent counters. This is done by - * specifying all columns which uniquely identify a time series. These are typically the counter - * name and any attributes tied to the counter. - * - * Use the ORDER BY clause of the window to order the observations by the associated timestamp - * in ascending order. - * - * @param value - * A cumulative counter. Must be a numeric data type. Must be non-negative. - * - * @param startTime - * A timestamp indicating when the counter was last set to zero. Used to signal counter - * resets. + * Extracts a part of the date/timestamp or interval source. * + * @param field + * selects which part of the source should be extracted, and supported string values are as + * same as the fields of the equivalent function `extract`. + * @param source + * a date/timestamp or time or interval column from where `field` should be extracted. * @return - * The difference between the current and previous counter value within the window partition, - * according to the order defined by the window's ORDER BY clause. Returns a column of the - * same type as the input. - * @group window_funcs - * @since 4.3.0 + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. + * @group datetime_funcs + * @since 3.5.0 */ - def counter_diff(value: Column, startTime: Column): Column = - Column.fn("counter_diff", value, startTime) + def date_part(field: Column, source: Column): Column = { + Column.fn("date_part", field, source) + } /** - * Window function: returns the cumulative distribution of values within a window partition, - * i.e. the fraction of rows that are below the current row. - * - * {{{ - * N = total number of rows in the partition - * cumeDist(x) = number of values before (and including) x / N - * }}} + * Extracts a part of the date/timestamp or interval source. * - * @group window_funcs - * @since 1.6.0 + * @param field + * selects which part of the source should be extracted, and supported string values are as + * same as the fields of the equivalent function `EXTRACT`. + * @param source + * a date/timestamp or interval column from where `field` should be extracted. * @return - * Returns a column that evaluates to a double. + * a part of the date/timestamp or interval source. Returns a column whose type depends on the + * field to extract, e.g. an integer for `YEAR` and a decimal for `SECOND`. + * @group datetime_funcs + * @since 3.5.0 */ - def cume_dist(): Column = Column.fn("cume_dist") + def datepart(field: Column, source: Column): Column = { + Column.fn("datepart", field, source) + } /** - * Window function: returns the rank of rows within a window partition, without any gaps. - * - * The difference between rank and dense_rank is that denseRank leaves no gaps in ranking - * sequence when there are ties. That is, if you were ranking a competition using dense_rank and - * had three people tie for second place, you would say that all three were in second place and - * that the next person came in third. Rank would give me sequential numbers, making the person - * that came in third place (after the ties) would register as coming in fifth. - * - * This is equivalent to the DENSE_RANK function in SQL. + * Returns the last day of the month which the given date belongs to. For example, input + * "2015-07-27" returns "2015-07-31" since July 31 is the last day of the month in July 2015. * - * @group window_funcs - * @since 1.6.0 + * @param e + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. * @return - * Returns a column that evaluates to an integer. + * A date, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def dense_rank(): Column = Column.fn("dense_rank") + def last_day(e: Column): Column = Column.fn("last_day", e) /** - * Window function: returns the value that is `offset` rows before the current row, and `null` - * if there is less than `offset` rows before the current row. For example, an `offset` of one - * will return the previous row at any given point in the window partition. - * - * This is equivalent to the LAG function in SQL. - * + * Extracts the minutes as an integer from a given date/time/timestamp/string. The input may + * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in + * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * The column to extract the minutes from. A column that evaluates to a date, time, timestamp + * or string. * @return - * Returns a column of the same type as the input. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def lag(e: Column, offset: Int): Column = lag(e, offset, null) + def minute(e: Column): Column = Column.fn("minute", e) /** - * Window function: returns the value that is `offset` rows before the current row, and `null` - * if there is less than `offset` rows before the current row. For example, an `offset` of one - * will return the previous row at any given point in the window partition. - * - * This is equivalent to the LAG function in SQL. + * Returns the day of the week for date/timestamp (0 = Monday, 1 = Tuesday, ..., 6 = Sunday). * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * @param e + * The column to extract the day of the week from. A column that evaluates to a date, + * timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to an integer. */ - def lag(columnName: String, offset: Int): Column = lag(columnName, offset, null) + def weekday(e: Column): Column = Column.fn("weekday", e) /** - * Window function: returns the value that is `offset` rows before the current row, and - * `defaultValue` if there is less than `offset` rows before the current row. For example, an - * `offset` of one will return the previous row at any given point in the window partition. - * - * This is equivalent to the LAG function in SQL. - * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 + * @param year + * The year to build the date. A column that evaluates to an integral. + * @param month + * The month to build the date. A column that evaluates to an integral. + * @param day + * The day to build the date. A column that evaluates to an integral. * @return - * Returns a column of the same type as the input. + * A date created from year, month and day fields. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 3.3.0 */ - def lag(columnName: String, offset: Int, defaultValue: Any): Column = { - lag(Column(columnName), offset, defaultValue) - } + def make_date(year: Column, month: Column, day: Column): Column = + Column.fn("make_date", year, month, day) /** - * Window function: returns the value that is `offset` rows before the current row, and - * `defaultValue` if there is less than `offset` rows before the current row. For example, an - * `offset` of one will return the previous row at any given point in the window partition. + * Returns number of months between dates `start` and `end`. * - * This is equivalent to the LAG function in SQL. + * A whole number is returned if both inputs have the same day of month or both are the last day + * of their respective months. Otherwise, the difference is calculated assuming 31 days per + * month. * - * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 - * @return - * Returns a column of the same type as the input. - */ - def lag(e: Column, offset: Int, defaultValue: Any): Column = { - lag(e, offset, defaultValue, false) - } - - /** - * Window function: returns the value that is `offset` rows before the current row, and - * `defaultValue` if there is less than `offset` rows before the current row. `ignoreNulls` - * determines whether null values of row are included in or eliminated from the calculation. For - * example, an `offset` of one will return the previous row at any given point in the window - * partition. - * - * This is equivalent to the LAG function in SQL. + * For example: + * {{{ + * months_between("2017-11-14", "2017-07-14") // returns 4.0 + * months_between("2017-01-01", "2017-01-10") // returns 0.29032258 + * months_between("2017-06-01", "2017-06-16 12:00:00") // returns -0.5 + * }}} * - * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @param ignoreNulls - * whether to ignore null values. A column that evaluates to a boolean. Must be a constant. - * @group window_funcs - * @since 3.2.0 + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can cast to a + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * timestamp. * @return - * Returns a column of the same type as the input. + * A double, or null if either `end` or `start` were strings that could not be cast to a + * timestamp. Negative if `end` is before `start`. Returns a column that evaluates to a + * double. + * @group datetime_funcs + * @since 1.5.0 */ - def lag(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = - Column.fn("lag", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) + def months_between(end: Column, start: Column): Column = + Column.fn("months_between", end, start) /** - * Window function: returns the value that is `offset` rows after the current row, and `null` if - * there is less than `offset` rows after the current row. For example, an `offset` of one will - * return the next row at any given point in the window partition. - * - * This is equivalent to the LEAD function in SQL. - * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * Returns number of months between dates `end` and `start`. If `roundOff` is set to true, the + * result is rounded off to 8 digits; it is not rounded otherwise. + * @param end + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param start + * A date, timestamp or string. If a string, the data must be in a format that can cast to a + * timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * timestamp. + * @param roundOff + * Whether to round off the result to 8 digits. A column that evaluates to a boolean. Must be + * a constant. + * @group datetime_funcs + * @since 2.4.0 * @return - * Returns a column of the same type as the input. + * Returns a column that evaluates to a double. */ - def lead(columnName: String, offset: Int): Column = { lead(columnName, offset, null) } + def months_between(end: Column, start: Column, roundOff: Boolean): Column = + Column.fn("months_between", end, start, lit(roundOff)) /** - * Window function: returns the value that is `offset` rows after the current row, and `null` if - * there is less than `offset` rows after the current row. For example, an `offset` of one will - * return the next row at any given point in the window partition. + * Returns the first date which is later than the value of the `date` column that is on the + * specified day of the week. * - * This is equivalent to the LEAD function in SQL. + * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first + * Sunday after 2015-07-27. * - * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param dayOfWeek + * Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", "Fri", "Sat", "Sun". A column + * that evaluates to a string. * @return - * Returns a column of the same type as the input. + * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` + * was an invalid value. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def lead(e: Column, offset: Int): Column = { lead(e, offset, null) } + def next_day(date: Column, dayOfWeek: String): Column = next_day(date, lit(dayOfWeek)) /** - * Window function: returns the value that is `offset` rows after the current row, and - * `defaultValue` if there is less than `offset` rows after the current row. For example, an - * `offset` of one will return the next row at any given point in the window partition. + * Returns the first date which is later than the value of the `date` column that is on the + * specified day of the week. * - * This is equivalent to the LEAD function in SQL. + * For example, `next_day('2015-07-27', "Sunday")` returns 2015-08-02 because that is the first + * Sunday after 2015-07-27. * - * @param columnName - * name of column or expression. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param dayOfWeek + * A column of the day of week. Case insensitive, and accepts: "Mon", "Tue", "Wed", "Thu", + * "Fri", "Sat", "Sun". A column that evaluates to a string. * @return - * Returns a column of the same type as the input. + * A date, or null if `date` was a string that could not be cast to a date or if `dayOfWeek` + * was an invalid value. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 3.2.0 */ - def lead(columnName: String, offset: Int, defaultValue: Any): Column = { - lead(Column(columnName), offset, defaultValue) - } + def next_day(date: Column, dayOfWeek: Column): Column = + Column.fn("next_day", date, dayOfWeek) /** - * Window function: returns the value that is `offset` rows after the current row, and - * `defaultValue` if there is less than `offset` rows after the current row. For example, an - * `offset` of one will return the next row at any given point in the window partition. - * - * This is equivalent to the LEAD function in SQL. - * + * Extracts the seconds as an integer from a given date/time/timestamp/string. The input may + * also be a nanosecond-precision timestamp `TIMESTAMP_NTZ(p)` or `TIMESTAMP_LTZ(p)` (`p` in + * `[7, 9]`, since 4.3.0), in which case the sub-microsecond digits are ignored. * @param e - * the column to compute on. A column of any type. - * @param offset - * number of rows to extend. A column that evaluates to an integer. Must be a constant. - * @param defaultValue - * default value. A column of any type. - * @group window_funcs - * @since 1.4.0 + * The column to extract the seconds from. A column that evaluates to a date, time, timestamp + * or string. * @return - * Returns a column of the same type as the input. + * An integer, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def lead(e: Column, offset: Int, defaultValue: Any): Column = { - lead(e, offset, defaultValue, false) - } + def second(e: Column): Column = Column.fn("second", e) /** - * Window function: returns the value that is `offset` rows after the current row, and - * `defaultValue` if there is less than `offset` rows after the current row. `ignoreNulls` - * determines whether null values of row are included in or eliminated from the calculation. The - * default value of `ignoreNulls` is false. For example, an `offset` of one will return the next - * row at any given point in the window partition. + * Extracts the week number as an integer from a given date/timestamp/string. * - * This is equivalent to the LEAD function in SQL. + * A week is considered to start on a Monday and week 1 is the first week with more than 3 days, + * as defined by ISO 8601 * * @param e - * The column to compute the lead value for. A column of any type. - * @param offset - * Number of rows after the current row to look ahead. A column that evaluates to an integral. - * Must be a constant. - * @param defaultValue - * Value to return when there are fewer than `offset` rows after the current row. A column of - * any type. Must be a constant. - * @param ignoreNulls - * Whether to skip null values when computing the result. A column that evaluates to a - * boolean. Must be a constant. - * @group window_funcs - * @since 3.2.0 + * The column to extract the week number from. A column that evaluates to a date, timestamp or + * string. * @return - * Returns a column of the same type as the input. + * An integer, or null if the input was a string that could not be cast to a date. Returns a + * column that evaluates to an integer. + * @group datetime_funcs + * @since 1.5.0 */ - def lead(e: Column, offset: Int, defaultValue: Any, ignoreNulls: Boolean): Column = - Column.fn("lead", false, e, lit(offset), lit(defaultValue), lit(ignoreNulls)) + def weekofyear(e: Column): Column = Column.fn("weekofyear", e) /** - * Window function: returns the value that is the `offset`th row of the window frame (counting - * from 1), and `null` if the size of window frame is less than `offset` rows. - * - * It will return the `offset`th non-null value it sees when ignoreNulls is set to true. If all - * values are null, then null is returned. - * - * This is equivalent to the nth_value function in SQL. + * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string + * representing the timestamp of that moment in the current system time zone in the yyyy-MM-dd + * HH:mm:ss format. * - * @param e - * The column to extract the value from. A column of any type. - * @param offset - * The 1-based row number within the window frame to use as the value. A column that evaluates - * to an integral. Must be a constant. - * @param ignoreNulls - * Whether the nth value should skip nulls when determining which row to use. A column that - * evaluates to a boolean. Must be a constant. - * @group window_funcs - * @since 3.1.0 + * @param ut + * A number of a type that is castable to a long, such as string or integer. Can be negative + * for timestamps before the unix epoch * @return - * Returns a column of the same type as the input. + * A string, or null if the input was a string that could not be cast to a long. Returns a + * column that evaluates to a string. + * @group datetime_funcs + * @since 1.5.0 */ - def nth_value(e: Column, offset: Int, ignoreNulls: Boolean): Column = - Column.fn("nth_value", false, e, lit(offset), lit(ignoreNulls)) + def from_unixtime(ut: Column): Column = Column.fn("from_unixtime", ut) /** - * Window function: returns the value that is the `offset`th row of the window frame (counting - * from 1), and `null` if the size of window frame is less than `offset` rows. + * Converts the number of seconds from unix epoch (1970-01-01 00:00:00 UTC) to a string + * representing the timestamp of that moment in the current system time zone in the given + * format. * - * This is equivalent to the nth_value function in SQL. + * See Datetime + * Patterns for valid date and time format patterns * - * @param e - * The column to extract the value from. A column of any type. - * @param offset - * The 1-based row number within the window frame to use as the value. A column that evaluates - * to an integral. Must be a constant. - * @group window_funcs - * @since 3.1.0 + * @param ut + * A number of a type that is castable to a long, such as string or integer. Can be negative + * for timestamps before the unix epoch + * @param f + * A date time pattern that the input will be formatted to * @return - * Returns a column of the same type as the input. + * A string, or null if `ut` was a string that could not be cast to a long or `f` was an + * invalid date time pattern. Returns a column that evaluates to a string. + * @group datetime_funcs + * @since 1.5.0 */ - def nth_value(e: Column, offset: Int): Column = nth_value(e, offset, false) + def from_unixtime(ut: Column, f: String): Column = + Column.fn("from_unixtime", ut, lit(f)) /** - * Window function: returns the ntile group id (from 1 to `n` inclusive) in an ordered window - * partition. For example, if `n` is 4, the first quarter of the rows will get value 1, the - * second quarter will get 2, the third quarter will get 3, and the last quarter will get 4. + * Returns the current Unix timestamp (in seconds) as a long. * - * This is equivalent to the NTILE function in SQL. + * @note + * All calls of `unix_timestamp` within the same query return the same value (i.e. the current + * timestamp is calculated at the start of query evaluation). * - * @param n - * The number of groups to divide the window partition into. A column that evaluates to an - * integral. Must be a constant. - * @group window_funcs - * @since 1.4.0 + * @group datetime_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a long. */ - def ntile(n: Int): Column = Column.fn("ntile", lit(n)) + def unix_timestamp(): Column = unix_timestamp(current_timestamp()) /** - * Window function: returns the relative rank (i.e. percentile) of rows within a window - * partition. - * - * This is computed by: - * {{{ - * (rank of row in its partition - 1) / (number of rows in the partition - 1) - * }}} - * - * This is equivalent to the PERCENT_RANK function in SQL. + * Converts time string in format yyyy-MM-dd HH:mm:ss to Unix timestamp (in seconds), using the + * default timezone and the default locale. * - * @group window_funcs - * @since 1.6.0 + * @param s + * A date, timestamp or string. If a string, the data must be in the `yyyy-MM-dd HH:mm:ss` + * format * @return - * Returns a column that evaluates to a double. + * A long, or null if the input was a string not of the correct format. Returns a column that + * evaluates to a long. + * @group datetime_funcs + * @since 1.5.0 */ - def percent_rank(): Column = Column.fn("percent_rank") + def unix_timestamp(s: Column): Column = Column.fn("unix_timestamp", s) /** - * Window function: returns the rank of rows within a window partition. - * - * The difference between rank and dense_rank is that dense_rank leaves no gaps in ranking - * sequence when there are ties. That is, if you were ranking a competition using dense_rank and - * had three people tie for second place, you would say that all three were in second place and - * that the next person came in third. Rank would give me sequential numbers, making the person - * that came in third place (after the ties) would register as coming in fifth. + * Converts time string with given pattern to Unix timestamp (in seconds). * - * This is equivalent to the RANK function in SQL. + * See Datetime + * Patterns for valid date and time format patterns * - * @group window_funcs - * @since 1.4.0 + * @param s + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * string, date, or timestamp. + * @param p + * A date time pattern detailing the format of `s` when `s` is a string. A column that + * evaluates to a string. * @return - * Returns a column that evaluates to an integer. + * A long, or null if `s` was a string that could not be cast to a date or `p` was an invalid + * format. Returns a column that evaluates to a long. + * @group datetime_funcs + * @since 1.5.0 */ - def rank(): Column = Column.fn("rank") + def unix_timestamp(s: Column, p: String): Column = + Column.fn("unix_timestamp", s, lit(p)) /** - * Window function: returns a sequential number starting at 1 within a window partition. + * Parses a string value to a time value. * - * @group window_funcs - * @since 1.6.0 + * @param str + * A string to be parsed to time. A column that evaluates to a string. * @return - * Returns a column that evaluates to an integer. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. + * + * @group datetime_funcs + * @since 4.1.0 */ - def row_number(): Column = Column.fn("row_number") - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Generator Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def to_time(str: Column): Column = { + Column.fn("to_time", str) + } /** - * Separates `col1`, ..., `colk` into `n` rows. Uses column names col0, col1, etc. by default - * unless specified otherwise. + * Parses a string value to a time value. * - * @param cols - * The first column must be a constant integer for the number of rows, and the remaining - * columns are the input elements to be separated into rows. - * @group generator_funcs - * @since 3.5.0 + * See Datetime + * Patterns for valid time format patterns. + * + * @param str + * A string to be parsed to time. + * @param format + * A time format pattern to follow. A column that evaluates to a string. * @return - * Returns a column of the same type as the input. + * A time, or raises an error if the input is malformed. Returns a column that evaluates to a + * time. + * @group datetime_funcs + * @since 4.1.0 */ - @scala.annotation.varargs - def stack(cols: Column*): Column = Column.fn("stack", cols: _*) + def to_time(str: Column, format: Column): Column = { + Column.fn("to_time", str, format) + } /** - * Creates a new row for each element in the given array or map column. Uses the default column - * name `col` for elements in the array and `key` and `value` for elements in the map unless - * specified otherwise. + * Converts to a timestamp by casting rules to `TimestampType`. * - * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 1.3.0 + * @param s + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. * @return - * Returns a column of the element type of the input array, or the key and value columns of - * the input map. + * A timestamp, or null if the input was a string that could not be cast to a timestamp. + * Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 2.2.0 */ - def explode(e: Column): Column = Column.fn("explode", e) + def to_timestamp(s: Column): Column = Column.fn("to_timestamp", s) /** - * Creates a new row for each element in the given array or map column. Uses the default column - * name `col` for elements in the array and `key` and `value` for elements in the map unless - * specified otherwise. Unlike explode, if the array/map is null or empty then null is produced. + * Converts time string with the given pattern to timestamp. * - * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 2.2.0 + * See Datetime + * Patterns for valid date and time format patterns + * + * @param s + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a string, date, timestamp, or numeric. + * @param fmt + * A date time pattern detailing the format of `s` when `s` is a string. A column that + * evaluates to a string. * @return - * Returns a column of the element type of the input array, or the key and value columns of - * the input map. + * A timestamp, or null if `s` was a string that could not be cast to a timestamp or `fmt` was + * an invalid format. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 2.2.0 */ - def explode_outer(e: Column): Column = Column.fn("explode_outer", e) + def to_timestamp(s: Column, fmt: String): Column = Column.fn("to_timestamp", s, lit(fmt)) /** - * Creates a new row for each element with position in the given array or map column. Uses the - * default column name `pos` for position, and `col` for elements in the array and `key` and - * `value` for elements in the map unless specified otherwise. + * Parses a string value to a time value. * - * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 2.1.0 + * @param str + * A string to be parsed to time. A column that evaluates to a string. * @return - * Returns the position column and a column of the element type of the input array, or the - * position column and the key and value columns of the input map. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. + * + * @group datetime_funcs + * @since 4.1.0 */ - def posexplode(e: Column): Column = Column.fn("posexplode", e) + def try_to_time(str: Column): Column = { + Column.fn("try_to_time", str) + } /** - * Creates a new row for each element with position in the given array or map column. Uses the - * default column name `pos` for position, and `col` for elements in the array and `key` and - * `value` for elements in the map unless specified otherwise. Unlike posexplode, if the - * array/map is null or empty then the row (null, null) is produced. + * Parses a string value to a time value. * - * @param e - * the target column to explode. A column that evaluates to an array or a map. - * @group generator_funcs - * @since 2.2.0 + * See Datetime + * Patterns for valid time format patterns. + * + * @param str + * A string to be parsed to time. + * @param format + * A time format pattern to follow. A column that evaluates to a string. * @return - * Returns the position column and a column of the element type of the input array, or the - * position column and the key and value columns of the input map. + * A time, or null if the input is malformed. Returns a column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 */ - def posexplode_outer(e: Column): Column = Column.fn("posexplode_outer", e) + def try_to_time(str: Column, format: Column): Column = { + Column.fn("try_to_time", str, format) + } /** - * Creates a new row for each element in the given array of structs. + * Parses the `s` with the `format` to a timestamp. The function always returns null on an + * invalid input with`/`without ANSI SQL mode enabled. The result data type is consistent with + * the value of configuration `spark.sql.timestampType`. * - * @param e - * the target column to explode. A column that evaluates to an array of structs. - * @group generator_funcs - * @since 3.4.0 + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. + * @param format + * Format to use to convert timestamp values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a timestamp. */ - def inline(e: Column): Column = Column.fn("inline", e) + def try_to_timestamp(s: Column, format: Column): Column = + Column.fn("try_to_timestamp", s, format) /** - * Creates a new row for each element in the given array of structs. Unlike inline, if the array - * is null or empty then null is produced for each nested column. + * Parses the `s` to a timestamp. The function always returns null on an invalid input + * with`/`without ANSI SQL mode enabled. It follows casting rules to a timestamp. The result + * data type is consistent with the value of configuration `spark.sql.timestampType`. * - * @param e - * the target column to explode. A column that evaluates to an array of structs. - * @group generator_funcs - * @since 3.4.0 + * @param s + * Column values to convert. A column that evaluates to a string, date, timestamp, or numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a timestamp. */ - def inline_outer(e: Column): Column = Column.fn("inline_outer", e) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Partition Transformation Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def try_to_timestamp(s: Column): Column = Column.fn("try_to_timestamp", s) /** - * (Java-specific) A transform for timestamps and dates to partition data into years. + * Converts the column into `DateType` by casting rules to `DateType`. * * @param e - * the target column to transform. A column that evaluates to a date or a timestamp. - * @group partition_transforms - * @since 3.0.0 + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @group datetime_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a date. */ - def years(e: Column): Column = partitioning.years(e) + def to_date(e: Column): Column = Column.fn("to_date", e) /** - * (Java-specific) A transform for timestamps and dates to partition data into months. + * Converts the column into a `DateType` with a specified format * - * @param e - * the target column to transform. A column that evaluates to a date or a timestamp. - * @group partition_transforms - * @since 3.0.0 + * See Datetime + * Patterns for valid date and time format patterns + * + * @param e + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * string, date, or timestamp. + * @param fmt + * A date time pattern detailing the format of `e` when `e`is a string. A column that + * evaluates to a string. + * @return + * A date, or null if `e` was a string that could not be cast to a date or `fmt` was an + * invalid format. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 2.2.0 */ - def months(e: Column): Column = partitioning.months(e) + def to_date(e: Column, fmt: String): Column = Column.fn("to_date", e, lit(fmt)) /** - * (Java-specific) A transform for timestamps and dates to partition data into days. + * This is a special version of `to_date` that performs the same operation, but returns a NULL + * value instead of raising an error if date cannot be created. * * @param e - * the target column to transform. A column that evaluates to a date or a timestamp. - * @group partition_transforms - * @since 3.0.0 + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @group datetime_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ - def days(e: Column): Column = partitioning.days(e) + def try_to_date(e: Column): Column = Column.fn("try_to_date", e) /** - * (Java-specific) A transform for timestamps to partition data into hours. + * This is a special version of `to_date` that performs the same operation, but returns a NULL + * value instead of raising an error if date cannot be created. * * @param e - * target date or timestamp column to work on. A column that evaluates to a date or timestamp. - * @group partition_transforms - * @since 3.0.0 + * Input column of values to convert. A column that evaluates to a string, date, or timestamp. + * @param fmt + * Format to use to convert date values. A column that evaluates to a string. Must be a + * constant. + * @group datetime_funcs + * @since 4.1.0 + * @return + * Returns a column that evaluates to a date. */ - def hours(e: Column): Column = partitioning.hours(e) + def try_to_date(e: Column, fmt: String): Column = Column.fn("try_to_date", e, lit(fmt)) /** - * (Java-specific) A transform for any type that partitions by a hash of the input column. + * Returns the number of days since 1970-01-01. * - * @param numBuckets - * The number of buckets. A column that evaluates to an integral. Must be a constant. * @param e - * The input column to partition. A column of any type. - * @group partition_transforms - * @since 3.0.0 + * Input column of values to convert. A column that evaluates to a date. + * @group datetime_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to an integer. */ - def bucket(numBuckets: Column, e: Column): Column = partitioning.bucket(numBuckets, e) + def unix_date(e: Column): Column = Column.fn("unix_date", e) /** - * (Java-specific) A transform for any type that partitions by a hash of the input column. + * Returns the number of microseconds since 1970-01-01 00:00:00 UTC. * - * @param numBuckets - * The number of buckets. Must be a constant. * @param e - * The input column to partition. A column of any type. - * @group partition_transforms - * @since 3.0.0 + * Input column of values to convert. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a long. */ - def bucket(numBuckets: Int, e: Column): Column = partitioning.bucket(numBuckets, e) - - // scalastyle:off - // TODO(SPARK-45970): Use @static annotation so Java can access to those - // API in the same way. Once we land this fix, should deprecate - // functions.hours, days, months, years and bucket. - object partitioning { - // scalastyle:on - /** - * (Scala-specific) A transform for timestamps and dates to partition data into years. - * - * @group partition_transforms - * @since 4.0.0 - */ - def years(e: Column): Column = Column.internalFn("years", e) - - /** - * (Scala-specific) A transform for timestamps and dates to partition data into months. - * - * @group partition_transforms - * @since 4.0.0 - */ - def months(e: Column): Column = Column.internalFn("months", e) - - /** - * (Scala-specific) A transform for timestamps and dates to partition data into days. - * - * @group partition_transforms - * @since 4.0.0 - */ - def days(e: Column): Column = Column.internalFn("days", e) - - /** - * (Scala-specific) A transform for timestamps to partition data into hours. - * - * @group partition_transforms - * @since 4.0.0 - */ - def hours(e: Column): Column = Column.internalFn("hours", e) - - /** - * (Scala-specific) A transform for any type that partitions by a hash of the input column. - * - * @group partition_transforms - * @since 4.0.0 - */ - def bucket(numBuckets: Column, e: Column): Column = Column.internalFn("bucket", numBuckets, e) - - /** - * (Scala-specific) A transform for any type that partitions by a hash of the input column. - * - * @group partition_transforms - * @since 4.0.0 - */ - def bucket(numBuckets: Int, e: Column): Column = bucket(lit(numBuckets), e) - } - - ////////////////////////////////////////////////////////////////////////////////////////////// - // CSV Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def unix_micros(e: Column): Column = Column.fn("unix_micros", e) - // scalastyle:off line.size.limit /** - * Parses a column containing a CSV string into a `StructType` with the specified schema. - * Returns `null`, in the case of an unparseable string. + * Returns the number of nanoseconds since 1970-01-01 00:00:00 UTC for a nanosecond-precision + * timestamp (`TIMESTAMP_LTZ(p)` / `TIMESTAMP_NTZ(p)`, `p` in `[7, 9]`). The result is a + * lossless `DECIMAL(21, 0)`. * * @param e - * a string column containing CSV data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the CSV string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the CSV is parsed. accepts the same options and the CSV data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * - * @group csv_funcs - * @since 3.0.0 + * input column of nanosecond-precision timestamp values to convert. A column that evaluates + * to a timestamp. + * @group datetime_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a decimal. */ - // scalastyle:on line.size.limit - def from_csv(e: Column, schema: StructType, options: Map[String, String]): Column = - from_csv(e, lit(schema.toDDL), options.iterator) + def unix_nanos(e: Column): Column = Column.fn("unix_nanos", e) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a CSV string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * Returns the number of milliseconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of + * precision. * * @param e - * a string column containing CSV data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the CSV string. A column that evaluates to a string. - * @param options - * options to control how the CSV is parsed. accepts the same options and the CSV data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * - * @group csv_funcs - * @since 3.0.0 + * input column of values to convert. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to a long. */ - // scalastyle:on line.size.limit - def from_csv(e: Column, schema: Column, options: java.util.Map[String, String]): Column = - from_csv(e, schema, options.asScala.iterator) - - private def from_csv(e: Column, schema: Column, options: Iterator[(String, String)]): Column = - Column.fnWithOptions("from_csv", options, e, schema) + def unix_millis(e: Column): Column = Column.fn("unix_millis", e) /** - * Parses a CSV string and infers its schema in DDL format. - * - * @param csv - * a CSV string. A string. Must be a constant. + * Returns the number of seconds since 1970-01-01 00:00:00 UTC. Truncates higher levels of + * precision. * - * @group csv_funcs - * @since 3.0.0 + * @param e + * input column of values to convert. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a long. */ - def schema_of_csv(csv: String): Column = schema_of_csv(lit(csv)) + def unix_seconds(e: Column): Column = Column.fn("unix_seconds", e) /** - * Parses a CSV string and infers its schema in DDL format. + * Returns date truncated to the unit specified by the format. * - * @param csv - * a foldable string column containing a CSV string. A column that evaluates to a string. + * For example, `trunc("2018-11-19 12:01:19", "year")` returns 2018-01-01 + * + * @param date + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a date, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to a + * date. + * @param format: + * 'year', 'yyyy', 'yy' to truncate by year, or 'month', 'mon', 'mm' to truncate by month + * Other options are: 'week', 'quarter'. A column that evaluates to a string. * - * @group csv_funcs - * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * A date, or null if `date` was a string that could not be cast to a date or `format` was an + * invalid value. Returns a column that evaluates to a date. + * @group datetime_funcs + * @since 1.5.0 */ - def schema_of_csv(csv: Column): Column = schema_of_csv(csv, Collections.emptyMap()) + def trunc(date: Column, format: String): Column = Column.fn("trunc", date, lit(format)) - // scalastyle:off line.size.limit /** - * Parses a CSV string and infers its schema in DDL format using options. + * Returns timestamp truncated to the unit specified by the format. * - * @param csv - * a foldable string column containing a CSV string. A column that evaluates to a string. - * @param options - * options to control how the CSV is parsed. accepts the same options and the CSV data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * For example, `date_trunc("year", "2018-11-19 12:01:19")` returns 2018-01-01 00:00:00 + * + * @param format: + * 'year', 'yyyy', 'yy' to truncate by year, 'month', 'mon', 'mm' to truncate by month, 'day', + * 'dd' to truncate by day, Other options are: 'microsecond', 'millisecond', 'second', + * 'minute', 'hour', 'week', 'quarter'. A column that evaluates to a string. + * @param timestamp + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. * @return - * a column with string literal containing schema in DDL format. Returns a column that - * evaluates to a string. - * @group csv_funcs - * @since 3.0.0 + * A timestamp, or null if `timestamp` was a string that could not be cast to a timestamp or + * `format` was an invalid value. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 2.3.0 */ - // scalastyle:on line.size.limit - def schema_of_csv(csv: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("schema_of_csv", options.asScala.iterator, csv) + def date_trunc(format: String, timestamp: Column): Column = + Column.fn("date_trunc", lit(format), timestamp) - // scalastyle:off line.size.limit /** - * (Java-specific) Converts a column containing a `StructType` into a CSV string with the - * specified schema. Throws an exception, in the case of an unsupported type. - * - * @param e - * a column containing a struct. A column that evaluates to a string. - * @param options - * options to control how the struct column is converted into a CSV string. It accepts the - * same options and the CSV data source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders + * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 + * 03:40:00.0'. * - * @group csv_funcs - * @since 3.0.0 + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. It should be in + * the format of either region-based zone IDs or zone offsets. Region IDs must have the form + * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return - * Returns a column that evaluates to a string. + * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was + * an invalid value. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 1.5.0 */ - // scalastyle:on line.size.limit - def to_csv(e: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("to_csv", options.asScala.iterator, e) + def from_utc_timestamp(ts: Column, tz: String): Column = from_utc_timestamp(ts, lit(tz)) /** - * Converts a column containing a `StructType` into a CSV string with the specified schema. - * Throws an exception, in the case of an unsupported type. - * - * @param e - * a column containing a struct. A column that evaluates to a string. - * - * @group csv_funcs - * @since 3.0.0 + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in UTC, and renders + * that time as a timestamp in the given time zone. For example, 'GMT+1' would yield '2017-07-14 + * 03:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. + * @group datetime_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // JSON Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def from_utc_timestamp(ts: Column, tz: Column): Column = + Column.fn("from_utc_timestamp", ts, tz) /** - * Extracts json object from a json string based on json path specified, and returns json string - * of the extracted json object. It will return null if the input json string is invalid. + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time + * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield + * '2017-07-14 01:40:00.0'. * - * @param e - * the JSON string column. A column that evaluates to a string. - * @param path - * the JSON path to extract. A column that evaluates to a string. Must be a constant. - * @group json_funcs - * @since 1.6.0 + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. It should be in + * the format of either region-based zone IDs or zone offsets. Region IDs must have the form + * 'area/city', such as 'America/Los_Angeles'. Zone offsets must be in the format + * '(+|-)HH:mm', for example '-08:00' or '+01:00'. Also 'UTC' and 'Z' are supported as aliases + * of '+00:00'. Other short names are not recommended to use because they can be ambiguous. A + * column that evaluates to a string. * @return - * Returns a column that evaluates to a string. + * A timestamp, or null if `ts` was a string that could not be cast to a timestamp or `tz` was + * an invalid value. Returns a column that evaluates to a timestamp. + * @group datetime_funcs + * @since 1.5.0 */ - def get_json_object(e: Column, path: String): Column = - Column.fn("get_json_object", e, lit(path)) + def to_utc_timestamp(ts: Column, tz: String): Column = to_utc_timestamp(ts, lit(tz)) /** - * Creates a new row for a json column according to the given field names. - * - * @param json - * the JSON string column. A column that evaluates to a string. - * @param fields - * the field names to extract. A column that evaluates to a string. Must be a constant. - * @group json_funcs - * @since 1.6.0 + * Given a timestamp like '2017-07-14 02:40:00.0', interprets it as a time in the given time + * zone, and renders that time as a timestamp in UTC. For example, 'GMT+1' would yield + * '2017-07-14 01:40:00.0'. + * @param ts + * A date, timestamp or string. If a string, the data must be in a format that can be cast to + * a timestamp, such as `yyyy-MM-dd` or `yyyy-MM-dd HH:mm:ss.SSSS`. A column that evaluates to + * a timestamp. + * @param tz + * A string detailing the time zone ID that the input should be adjusted to. A column that + * evaluates to a string. + * @group datetime_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - @scala.annotation.varargs - def json_tuple(json: Column, fields: String*): Column = { - require(fields.nonEmpty, "at least 1 field name should be given.") - Column.fn("json_tuple", json +: fields.map(lit): _*) - } + def to_utc_timestamp(ts: Column, tz: Column): Column = Column.fn("to_utc_timestamp", ts, tz) - // scalastyle:off line.size.limit /** - * (Scala-specific) Parses a column containing a JSON string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * Bucketize rows into one or more time windows given a timestamp specifying column. Window + * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window + * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in + * the order of months are not supported. The following example takes the average stock price + * for a one minute window every 10 seconds starting 5 seconds after the hour: * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. Accepts the same options as the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * {{{ + * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType + * df.groupBy(window($"timestamp", "1 minute", "10 seconds", "5 seconds"), $"stockId") + * .agg(mean("price")) + * }}} * - * @group json_funcs - * @since 2.1.0 - * @return - * Returns a column that evaluates to a struct. - */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = - from_json(e, schema.asInstanceOf[DataType], options) - - // scalastyle:off line.size.limit - /** - * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. + * The windows will look like: * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * {{{ + * 09:00:05-09:01:05 + * 09:00:15-09:01:15 + * 09:00:25-09:01:25 ... + * }}} * - * @group json_funcs - * @since 2.2.0 - * @return - * Returns a column of the type given by the schema (a struct, array, or map). - */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { - from_json(e, lit(schema.sql), options.iterator) - } - - // scalastyle:off line.size.limit - /** - * (Java-specific) Parses a column containing a JSON string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param windowDuration + * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that + * the duration is a fixed length of time, and does not vary over time according to a + * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A + * column that evaluates to a string. + * @param slideDuration + * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will + * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. + * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This + * duration is likewise absolute, and does not vary according to a calendar. A column that + * evaluates to a string. + * @param startTime + * The offset with respect to 1970-01-01 00:00:00 UTC with which to start window intervals. + * For example, in order to have hourly tumbling windows that start 15 minutes past the hour, + * e.g. 12:15-13:15, 13:15-14:15... provide `startTime` as `15 minutes`. A column that + * evaluates to a string. * - * @group json_funcs - * @since 2.1.0 + * @group datetime_funcs + * @since 2.0.0 * @return * Returns a column that evaluates to a struct. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = - from_json(e, schema, options.asScala.toMap) + def window( + timeColumn: Column, + windowDuration: String, + slideDuration: String, + startTime: String): Column = + Column.fn("window", timeColumn, lit(windowDuration), lit(slideDuration), lit(startTime)) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. + * Bucketize rows into one or more time windows given a timestamp specifying column. Window + * starts are inclusive but the window ends are exclusive, e.g. 12:05 will be in the window + * [12:05,12:10) but not in [12:00,12:05). Windows can support microsecond precision. Windows in + * the order of months are not supported. The windows start beginning at 1970-01-01 00:00:00 + * UTC. The following example takes the average stock price for a one minute window every 10 + * seconds: * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * {{{ + * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType + * df.groupBy(window($"timestamp", "1 minute", "10 seconds"), $"stockId") + * .agg(mean("price")) + * }}} * - * @group json_funcs - * @since 2.2.0 + * The windows will look like: + * + * {{{ + * 09:00:00-09:01:00 + * 09:00:10-09:01:10 + * 09:00:20-09:01:20 ... + * }}} + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param windowDuration + * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. Note that + * the duration is a fixed length of time, and does not vary over time according to a + * calendar. For example, `1 day` always means 86,400,000 milliseconds, not a calendar day. A + * column that evaluates to a string. + * @param slideDuration + * A string specifying the sliding interval of the window, e.g. `1 minute`. A new window will + * be generated every `slideDuration`. Must be less than or equal to the `windowDuration`. + * Check `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. This + * duration is likewise absolute, and does not vary according to a calendar. A column that + * evaluates to a string. + * + * @group datetime_funcs + * @since 2.0.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a struct. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { - from_json(e, schema, options.asScala.toMap) + def window(timeColumn: Column, windowDuration: String, slideDuration: String): Column = { + window(timeColumn, windowDuration, slideDuration, "0 second") } /** - * Parses a column containing a JSON string into a `StructType` with the specified schema. - * Returns `null`, in the case of an unparseable string. + * Generates tumbling time windows given a timestamp specifying column. Window starts are + * inclusive but the window ends are exclusive, e.g. 12:05 will be in the window [12:05,12:10) + * but not in [12:00,12:05). Windows can support microsecond precision. Windows in the order of + * months are not supported. The windows start beginning at 1970-01-01 00:00:00 UTC. The + * following example takes the average stock price for a one minute tumbling window: * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. + * {{{ + * val df = ... // schema => timestamp: TimestampType, stockId: StringType, price: DoubleType + * df.groupBy(window($"timestamp", "1 minute"), $"stockId") + * .agg(mean("price")) + * }}} * - * @group json_funcs - * @since 2.1.0 + * The windows will look like: + * + * {{{ + * 09:00:00-09:01:00 + * 09:01:00-09:02:00 + * 09:02:00-09:03:00 ... + * }}} + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param windowDuration + * A string specifying the width of the window, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column + * that evaluates to a string. + * + * @group datetime_funcs + * @since 2.0.0 * @return * Returns a column that evaluates to a struct. */ - def from_json(e: Column, schema: StructType): Column = - from_json(e, schema, Map.empty[String, String]) + def window(timeColumn: Column, windowDuration: String): Column = { + window(timeColumn, windowDuration, windowDuration, "0 second") + } /** - * Parses a column containing a JSON string into a `MapType` with `StringType` as keys type, - * `StructType` or `ArrayType` with the specified schema. Returns `null`, in the case of an - * unparseable string. + * Extracts the event time from the window column. * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A string, StructType or DataType. Must be a - * constant. + * The window column is of StructType { start: Timestamp, end: Timestamp } where start is + * inclusive and end is exclusive. Since event time can support microsecond precision, + * window_time(window) = window.end - 1 microsecond. * - * @group json_funcs - * @since 2.2.0 + * @param windowColumn + * The window column (typically produced by window aggregation) of type StructType { start: + * Timestamp, end: Timestamp }. A column that evaluates to a struct. + * + * @group datetime_funcs + * @since 3.4.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a timestamp. */ - def from_json(e: Column, schema: DataType): Column = - from_json(e, schema, Map.empty[String, String]) + def window_time(windowColumn: Column): Column = Column.fn("window_time", windowColumn) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. + * Generates session window given a timestamp specifying column. * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Session window is one of dynamic windows, which means the length of window is varying + * according to the given inputs. The length of session window is defined as "the timestamp of + * latest input of the session + gap duration", so when the new inputs are bound to the current + * session window, the end time of session window can be expanded according to the new inputs. * - * @group json_funcs - * @since 2.1.0 + * Windows can support microsecond precision. gapDuration in the order of months are not + * supported. + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param gapDuration + * A string specifying the timeout of the session, e.g. `10 minutes`, `1 second`. Check + * `org.apache.spark.unsafe.types.CalendarInterval` for valid duration identifiers. A column + * that evaluates to a string. + * + * @group datetime_funcs + * @since 3.2.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a struct. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { - from_json(e, schema, options.asScala.toMap) - } + def session_window(timeColumn: Column, gapDuration: String): Column = + session_window(timeColumn, lit(gapDuration)) - // scalastyle:off line.size.limit /** - * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the - * case of an unparseable string. + * Generates session window given a timestamp specifying column. * - * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * Session window is one of dynamic windows, which means the length of window is varying + * according to the given inputs. For static gap duration, the length of session window is + * defined as "the timestamp of latest input of the session + gap duration", so when the new + * inputs are bound to the current session window, the end time of session window can be + * expanded according to the new inputs. * - * @group json_funcs - * @since 2.3.0 + * Besides a static gap duration value, users can also provide an expression to specify gap + * duration dynamically based on the input row. With dynamic gap duration, the closing of a + * session window does not depend on the latest input anymore. A session window's range is the + * union of all events' ranges which are determined by event start time and evaluated gap + * duration during the query execution. Note that the rows with negative or zero gap duration + * will be filtered out from the aggregation. + * + * Windows can support microsecond precision. gapDuration in the order of months are not + * supported. + * + * For a streaming query, you may use the function `current_timestamp` to generate windows on + * processing time. + * + * @param timeColumn + * The column or the expression to use as the timestamp for windowing by time. The time column + * must be of TimestampType or TimestampNTZType. A column that evaluates to a timestamp. + * @param gapDuration + * A column specifying the timeout of the session. It could be static value, e.g. `10 + * minutes`, `1 second`, or an expression/UDF that specifies gap duration dynamically based on + * the input row. A column that evaluates to a string or interval. + * + * @group datetime_funcs + * @since 3.2.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a struct. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: String, options: Map[String, String]): Column = { - from_json(e, lit(schema), options.asJava) - } + def session_window(timeColumn: Column, gapDuration: Column): Column = + Column.fn("session_window", timeColumn, gapDuration) /** - * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns - * `null`, in the case of an unparseable string. - * + * Converts the number of seconds from the Unix epoch (1970-01-01T00:00:00Z) to a timestamp. * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A column that evaluates to a string. - * - * @group json_funcs - * @since 2.4.0 + * unix time values. A column that evaluates to a numeric. + * @group datetime_funcs + * @since 3.1.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a timestamp. */ - def from_json(e: Column, schema: Column): Column = { - from_json(e, schema, Map.empty[String, String].asJava) - } + def timestamp_seconds(e: Column): Column = Column.fn("timestamp_seconds", e) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` - * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns - * `null`, in the case of an unparseable string. + * Creates timestamp from the number of milliseconds since UTC epoch. * * @param e - * a string column containing JSON data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the json string. A column that evaluates to a string. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * - * @group json_funcs - * @since 2.4.0 + * unix time values. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column of the type given by the schema (a struct, array, or map). + * Returns a column that evaluates to a timestamp. */ - // scalastyle:on line.size.limit - def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { - from_json(e, schema, options.asScala.iterator) - } - - private def from_json( - e: Column, - schema: Column, - options: Iterator[(String, String)]): Column = { - Column.fnWithOptions("from_json", options, e, schema) - } + def timestamp_millis(e: Column): Column = Column.fn("timestamp_millis", e) /** - * Parses a JSON string and infers its schema in DDL format. - * - * @param json - * a JSON string. A string. Must be a constant. + * Creates timestamp from the number of microseconds since UTC epoch. * - * @group json_funcs - * @since 2.4.0 + * @param e + * unix time values. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def schema_of_json(json: String): Column = schema_of_json(lit(json)) + def timestamp_micros(e: Column): Column = Column.fn("timestamp_micros", e) /** - * Parses a JSON string and infers its schema in DDL format. - * - * @param json - * a foldable string column containing a JSON string. A column that evaluates to a string. + * Creates a timestamp with the local time zone and nanosecond precision (TIMESTAMP_LTZ(9)) from + * the number of nanoseconds since UTC epoch. * - * @group json_funcs - * @since 2.4.0 + * @param e + * nanosecond values since the UTC epoch. A column that evaluates to an integral or decimal. + * @group datetime_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) + def timestamp_nanos(e: Column): Column = Column.fn("timestamp_nanos", e) - // scalastyle:off line.size.limit /** - * Parses a JSON string and infers its schema in DDL format using options. + * Gets the difference between the timestamps in the specified units by truncating the fraction + * part. * - * @param json - * a foldable string column containing JSON data. A column that evaluates to a string. - * @param options - * options to control how the json is parsed. accepts the same options and the json data - * source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. + * @param unit + * the units of the difference between the given timestamps, e.g. 'YEAR', 'MONTH', 'DAY', + * 'HOUR'. A column that evaluates to a string. Must be a constant. + * @param start + * A timestamp which the expression subtracts from `end`. A column that evaluates to a + * timestamp. + * @param end + * A timestamp from which the expression subtracts `start`. A column that evaluates to a + * timestamp. + * @group datetime_funcs + * @since 4.0.0 * @return - * a column with string literal containing schema in DDL format. Returns a column that - * evaluates to a string. - * - * @group json_funcs - * @since 3.0.0 + * Returns a column that evaluates to a long. */ - // scalastyle:on line.size.limit - def schema_of_json(json: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("schema_of_json", options.asScala.iterator, json) + def timestamp_diff(unit: String, start: Column, end: Column): Column = + Column.internalFn("timestampdiff", lit(unit), start, end) /** - * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of any - * other valid JSON string, `NULL` or an invalid JSON. + * Adds the specified number of units to the given timestamp. * - * @param e - * the JSON array string column. A column that evaluates to a string. - * @group json_funcs - * @since 3.5.0 + * @param unit + * the units of datetime to add, e.g. 'YEAR', 'MONTH', 'DAY', 'HOUR'. A column that evaluates + * to a string. Must be a constant. + * @param quantity + * the number of units of time to add. A column that evaluates to an integral. + * @param ts + * A timestamp to which to add. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the input. */ - def json_array_length(e: Column): Column = Column.fn("json_array_length", e) + def timestamp_add(unit: String, quantity: Column, ts: Column): Column = + Column.internalFn("timestampadd", lit(unit), quantity, ts) /** - * Returns all the keys of the outermost JSON object as an array. If a valid JSON object is - * given, all the keys of the outermost object will be returned as an array. If it is any other - * valid JSON string, an invalid JSON string or an empty string, the function returns null. + * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets + * aligned to the default origin (1970-01-01 00:00:00). For `TIMESTAMP_NTZ`, bucketing is + * performed in UTC. For `TIMESTAMP`, year-month interval buckets and calendar-day components of + * day-time interval buckets align to the session time zone. * - * @param e - * the JSON object string column. A column that evaluates to a string. - * @group json_funcs - * @since 3.5.0 + * @param bucketSize + * A day-time or year-month interval defining the bucket size. Must be positive and foldable. + * @param ts + * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to an array. + * Returns a column of the same type as the input. */ - def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) + def time_bucket(bucketSize: Column, ts: Column): Column = + Column.fn("time_bucket", bucketSize, ts) /** - * Returns the type of the outermost JSON value as a string: one of 'object', 'array', 'string', - * 'number', 'boolean', or 'null'. Returns null for invalid or empty input. + * Returns the start of the fixed-size bucket of `bucketSize` that contains `ts`, with buckets + * aligned to `origin`. For `TIMESTAMP_NTZ`, bucketing is performed in UTC. For `TIMESTAMP`, + * year-month interval buckets and calendar-day components of day-time interval buckets align to + * the session time zone. * - * @param e - * the JSON string column. A column that evaluates to a string. - * @group json_funcs - * @since 4.4.0 + * @param bucketSize + * A day-time or year-month interval defining the bucket size. Must be positive and foldable. + * @param ts + * A TIMESTAMP or TIMESTAMP_NTZ value to bucket. + * @param origin + * Alignment anchor. Must be the same type as `ts` and must be foldable. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the input. */ - def json_typeof(e: Column): Column = Column.fn("json_typeof", e) + def time_bucket(bucketSize: Column, ts: Column, origin: Column): Column = + Column.fn("time_bucket", bucketSize, ts, origin) - // scalastyle:off line.size.limit /** - * (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into - * a JSON string with the specified schema. Throws an exception, in the case of an unsupported - * type. - * - * @param e - * a column containing a struct, an array, a map, or a variant. A column that evaluates to a - * struct, array, map, or variant. - * @param options - * options to control how the struct column is converted into a json string. accepts the same - * options and the json data source. See Data - * Source Option in the version you use. Additionally the function supports the `pretty` - * option which enables pretty JSON generation. A map of string options. Must be a constant. + * Returns the difference between two times, measured in specified units. Throws a + * SparkIllegalArgumentException, in case the specified unit is not supported. * - * @group json_funcs - * @since 2.1.0 + * @param unit + * A STRING representing the unit of the time difference. Supported units are: "HOUR", + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. + * @param start + * A starting TIME. A column that evaluates to a time. + * @param end + * An ending TIME. A column that evaluates to a time. * @return - * Returns a column that evaluates to a string. + * The difference between `end` and `start` times, measured in specified units. Returns a + * column that evaluates to a long. + * @note + * If any of the inputs is `NULL`, the result is `NULL`. + * @group datetime_funcs + * @since 4.1.0 */ - // scalastyle:on line.size.limit - def to_json(e: Column, options: Map[String, String]): Column = - Column.fnWithOptions("to_json", options.iterator, e) + def time_diff(unit: Column, start: Column, end: Column): Column = { + Column.fn("time_diff", unit, start, end) + } - // scalastyle:off line.size.limit /** - * (Java-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into - * a JSON string with the specified schema. Throws an exception, in the case of an unsupported - * type. - * - * @param e - * a column containing a struct, an array, a map, or a variant. A column that evaluates to a - * struct, array, map, or variant. - * @param options - * options to control how the struct column is converted into a json string. accepts the same - * options and the json data source. See Data - * Source Option in the version you use. Additionally the function supports the `pretty` - * option which enables pretty JSON generation. A map of string options. Must be a constant. + * Returns `time` truncated to the `unit`. * - * @group json_funcs - * @since 2.1.0 + * @param unit + * A STRING representing the unit to truncate the time to. Supported units are: "HOUR", + * "MINUTE", "SECOND", "MILLISECOND", and "MICROSECOND". The unit is case-insensitive. A + * column that evaluates to a string. + * @param time + * A TIME to truncate. A column that evaluates to a time. * @return - * Returns a column that evaluates to a string. + * A TIME truncated to the specified unit. Returns a column that evaluates to a time. + * @note + * If any of the inputs is `NULL`, the result is `NULL`. + * @throws IllegalArgumentException + * If the `unit` is not supported. + * @group datetime_funcs + * @since 4.1.0 */ - // scalastyle:on line.size.limit - def to_json(e: Column, options: java.util.Map[String, String]): Column = - to_json(e, options.asScala.toMap) + def time_trunc(unit: Column, time: Column): Column = { + Column.fn("time_trunc", unit, time) + } /** - * Converts a column containing a `StructType`, `ArrayType` or a `MapType` into a JSON string - * with the specified schema. Throws an exception, in the case of an unsupported type. + * Creates a TIME from the number of seconds since midnight. * * @param e - * a column containing a struct, an array, a map, or a variant. A column that evaluates to a - * struct, array, map, or variant. - * - * @group json_funcs - * @since 2.1.0 + * seconds since midnight (0 to 86399.999999). A column that evaluates to a numeric. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a time. */ - def to_json(e: Column): Column = - to_json(e, Map.empty[String, String]) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // VARIANT Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def time_from_seconds(e: Column): Column = Column.fn("time_from_seconds", e) /** - * Parses a JSON string and constructs a Variant value. Returns null if the input string is not - * a valid JSON value. - * - * @param json - * a string column that contains JSON data. A column that evaluates to a string. + * Creates a TIME from the number of milliseconds since midnight. * - * @group variant_funcs - * @since 4.0.0 + * @param e + * milliseconds since midnight (0 to 86399999). A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a time. */ - def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) + def time_from_millis(e: Column): Column = Column.fn("time_from_millis", e) /** - * Parses a JSON string and constructs a Variant value. + * Creates a TIME from the number of microseconds since midnight. * - * @param json - * a string column that contains JSON data. A column that evaluates to a string. - * @group variant_funcs - * @since 4.0.0 + * @param e + * microseconds since midnight (0 to 86399999999). A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a time. */ - def parse_json(json: Column): Column = Column.fn("parse_json", json) + def time_from_micros(e: Column): Column = Column.fn("time_from_micros", e) /** - * Converts a column containing nested inputs (array/map/struct) into a variants where maps and - * structs are converted to variant objects which are unordered unlike SQL structs. Input maps - * can only have string keys. + * Extracts the number of seconds (including fractional seconds) from a TIME value. Returns a + * DECIMAL(14,6) to preserve microsecond precision. * - * @param col - * a column with a nested schema or column name. A column that evaluates to a struct, array, - * map, or variant. - * @group variant_funcs - * @since 4.0.0 + * @param e + * TIME value to convert. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a decimal. */ - def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) + def time_to_seconds(e: Column): Column = Column.fn("time_to_seconds", e) /** - * Creates a variant object from the given arrays of keys and values. The keys must be non-null - * strings and the two arrays must have the same length. + * Extracts the number of milliseconds since midnight from a TIME value. * - * @param keys - * a column that evaluates to an array of string keys. - * @param values - * a column that evaluates to an array of values. - * @group variant_funcs - * @since 4.4.0 + * @param e + * the TIME value to convert. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a long. */ - def variant_from_arrays(keys: Column, values: Column): Column = - Column.fn("variant_from_arrays", keys, values) + def time_to_millis(e: Column): Column = Column.fn("time_to_millis", e) /** - * Creates a variant object from an array of key/value struct entries. The keys must be non-null - * strings. + * Extracts the number of microseconds since midnight from a TIME value. * - * @param entries - * a column that evaluates to an array of key/value structs. - * @group variant_funcs - * @since 4.4.0 + * @param e + * the TIME value to convert. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a long. */ - def variant_from_entries(entries: Column): Column = - Column.fn("variant_from_entries", entries) + def time_to_micros(e: Column): Column = Column.fn("time_to_micros", e) /** - * Check if a variant value is a variant null. Returns true if and only if the input is a - * variant null and false otherwise (including in the case of SQL NULL). + * Parses the `timestamp` expression with the `format` expression to a timestamp with local time + * zone. Returns null with invalid input. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.0.0 + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a timestamp. */ - def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) + def to_timestamp_ltz(timestamp: Column, format: Column): Column = + Column.fn("to_timestamp_ltz", timestamp, format) /** - * Check if a variant value is valid. Returns true if the variant is valid, false if it is - * malformed, and NULL if the input is NULL. + * Parses the `timestamp` expression with the default format to a timestamp with local time + * zone. The default format follows casting rules to a timestamp. Returns null with invalid + * input. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.2.0 + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a boolean. + * Returns a column that evaluates to a timestamp. */ - def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) + def to_timestamp_ltz(timestamp: Column): Column = + Column.fn("to_timestamp_ltz", timestamp) /** - * Removes fields or array elements from a variant at the given JSONPath locations. Multiple - * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. + * Parses the `timestamp_str` expression with the `format` expression to a timestamp without + * time zone. Returns null with invalid input. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the first JSONPath string. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A column that evaluates to a string. - * @param paths - * additional JSONPath arguments, applied after `path` in order. A column that evaluates to a - * string. - * @group variant_funcs - * @since 5.0.0 + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to parse the timestamp values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a timestamp. */ - @scala.annotation.varargs - def variant_delete(v: Column, path: Column, paths: Column*): Column = - Column.fn("variant_delete", (v +: path +: paths): _*) + def to_timestamp_ntz(timestamp: Column, format: Column): Column = + Column.fn("to_timestamp_ntz", timestamp, format) /** - * Removes fields or array elements from a variant at the given JSONPath locations. Multiple - * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. + * Parses the `timestamp` expression with the default format to a timestamp without time zone. + * The default format follows casting rules to a timestamp. Returns null with invalid input. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the first JSONPath identifying a deletion target. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A string. Must be a constant. - * @param paths - * additional JSONPath strings, applied after `path` in order. A string. Must be a constant. - * @group variant_funcs - * @since 5.0.0 + * @param timestamp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a timestamp. */ - @scala.annotation.varargs - def variant_delete(v: Column, path: String, paths: String*): Column = - Column.fn("variant_delete", (v +: lit(path) +: paths.map(lit)): _*) + def to_timestamp_ntz(timestamp: Column): Column = + Column.fn("to_timestamp_ntz", timestamp) /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field (error if it already exists); an array path inserts at the index, shifting later - * elements right. Missing intermediate keys are created. Throws an error if a path segment hits - * a value of an incompatible type. Returns NULL if any argument is NULL. + * Returns the UNIX timestamp of the given time. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the insertion target. A valid path - * should start with `$` and is followed by one or more segments like `[123]`, `.name`, - * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a - * string. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @param format + * the format used to convert the time values. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a long. */ - def variant_insert(v: Column, path: Column, value: Column): Column = - Column.fn("variant_insert", v, path, value) + def to_unix_timestamp(timeExp: Column, format: Column): Column = + Column.fn("to_unix_timestamp", timeExp, format) /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field (error if it already exists); an array path inserts at the index, shifting later - * elements right. Missing intermediate keys are created. Throws an error if a path segment hits - * a value of an incompatible type. Returns NULL if any argument is NULL. + * Returns the UNIX timestamp of the given time. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the insertion target. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A string. Must be a constant. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param timeExp + * the input column or strings. A column that evaluates to a date, timestamp or string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a variant. + * Returns a column that evaluates to a long. */ - def variant_insert(v: Column, path: String, value: Column): Column = - Column.fn("variant_insert", v, lit(path), value) + def to_unix_timestamp(timeExp: Column): Column = + Column.fn("to_unix_timestamp", timeExp) /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field; an array path inserts at the index, shifting later elements right. Missing - * intermediate keys are created. Returns NULL if the field already exists or a path segment - * hits a value of an incompatible type, or if any argument is NULL. + * Extracts the three-letter abbreviated month name from a given date/timestamp/string. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the insertion target. A valid path - * should start with `$` and is followed by one or more segments like `[123]`, `.name`, - * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or * string. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @group datetime_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ - def try_variant_insert(v: Column, path: Column, value: Column): Column = - Column.fn("try_variant_insert", v, path, value) + def monthname(timeExp: Column): Column = + Column.fn("monthname", timeExp) /** - * Inserts a value into a variant at the given JSONPath location. An object path adds a new - * field; an array path inserts at the index, shifting later elements right. Missing - * intermediate keys are created. Returns NULL if the field already exists or a path segment - * hits a value of an incompatible type, or if any argument is NULL. + * Extracts the three-letter abbreviated day name from a given date/timestamp/string. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the insertion target. A valid path should start with `$` and is - * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root - * path `$` is not allowed. A string. Must be a constant. - * @param value - * the value to insert. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param timeExp + * the target date/timestamp to work on. A column that evaluates to a date, timestamp or + * string. + * @group datetime_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a string. */ - def try_variant_insert(v: Column, path: String, value: Column): Column = - Column.fn("try_variant_insert", v, lit(path), value) + def dayname(timeExp: Column): Column = + Column.fn("dayname", timeExp) - /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Throws an error if a path segment hits a value of an incompatible type. - * Returns NULL if any argument is NULL. - * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. - * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 - */ - def variant_set(v: Column, path: Column, value: Column): Column = - Column.fn("variant_set", v, path, value) + ////////////////////////////////////////////////////////////////////////////////////////////// + // Collection functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Throws an error if a path segment hits a value of an incompatible type. - * Returns NULL if any argument is NULL. - * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. A string. Must be a constant. + * Returns true if the array contains `value`, false if not. Returns null if the array or + * `value` is null, or if `value` is not found and the array contains a null element. + * @param column + * the target column containing the arrays. A column that evaluates to an array. * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * the value to check for in the array. A column that evaluates to a value matching the + * array's element type. + * @group array_funcs + * @since 1.5.0 + * @return + * Returns a column that evaluates to a boolean. */ - def variant_set(v: Column, path: String, value: Column): Column = - Column.fn("variant_set", v, lit(path), value) + def array_contains(column: Column, value: Any): Column = + Column.fn("array_contains", column, lit(value)) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns - * NULL if any argument is NULL. + * Returns an ARRAY containing all elements from the source ARRAY as well as the new element. + * The new element/column is located at end of the ARRAY. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. A boolean. Must be a - * constant. - * @group variant_funcs - * @since 4.3.0 + * @param column + * the source column containing the array. A column that evaluates to an array. + * @param element + * the value to append to the array. A column that evaluates to a value matching the array's + * element type. + * @group array_funcs + * @since 3.4.0 + * @return + * Returns a column that evaluates to an array. */ - def variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = - Column.fn("variant_set", v, path, value, lit(createIfMissing)) + def array_append(column: Column, element: Any): Column = + Column.fn("array_append", column, lit(element)) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns - * NULL if any argument is NULL. - * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. A string. Must be a constant. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. A boolean. Must be a - * constant. - * @group variant_funcs - * @since 4.3.0 + * Returns `true` if `a1` and `a2` have at least one non-null element in common. If not and both + * the arrays are non-empty and any of them contains a `null`, it returns `null`. It returns + * `false` otherwise. + * @param a1 + * the first input array. A column that evaluates to an array. + * @param a2 + * the second input array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 + * @return + * Returns a column that evaluates to a boolean. */ - def variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = - Column.fn("variant_set", v, lit(path), value, lit(createIfMissing)) + def arrays_overlap(a1: Column, a2: Column): Column = Column.fn("arrays_overlap", a1, a2) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if - * any argument is NULL. + * Returns an array containing all the elements in `x` from index `start` (or starting from the + * end if `start` is negative) with the specified `length`. * - * @param v - * a variant column. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 - */ - def try_variant_set(v: Column, path: Column, value: Column): Column = - Column.fn("try_variant_set", v, path, value) - - /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if - * any argument is NULL. + * @param x + * the array column to be sliced. A column that evaluates to an array. + * @param start + * the starting index. A column that evaluates to an integer. + * @param length + * the length of the slice. A column that evaluates to an integer. * - * @param v - * a variant column. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @group array_funcs + * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ - def try_variant_set(v: Column, path: String, value: Column): Column = - Column.fn("try_variant_set", v, lit(path), value) + def slice(x: Column, start: Int, length: Int): Column = + slice(x, lit(start), lit(length)) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any - * argument is NULL. + * Returns an array containing all the elements in `x` from index `start` (or starting from the + * end if `start` is negative) with the specified `length`. * - * @param v - * a variant column. - * @param path - * the column containing the JSONPath string identifying the set target. A valid path should - * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. The root path `$` is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. - * @group variant_funcs - * @since 4.3.0 + * @param x + * the array column to be sliced. A column that evaluates to an array. + * @param start + * the starting index. A column that evaluates to an integer. + * @param length + * the length of the slice. A column that evaluates to an integer. + * + * @group array_funcs + * @since 3.1.0 + * @return + * Returns a column that evaluates to an array. */ - def try_variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = - Column.fn("try_variant_set", v, path, value, lit(createIfMissing)) + def slice(x: Column, start: Column, length: Column): Column = + Column.fn("slice", x, start, length) /** - * Sets or upserts a value in a variant at the given JSONPath location. An existing object field - * or array element at the target is replaced. A missing field, array index, or intermediate - * path is created, unless `createIfMissing` is false, in which case the variant is left - * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any - * argument is NULL. + * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is + * negative or greater than the number of elements in the array. * - * @param v - * a variant column. - * @param path - * the JSONPath identifying the set target. A valid path should start with `$` and is followed - * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` - * is not allowed. - * @param value - * the value to set. Any expression castable to variant. - * @param createIfMissing - * whether to create missing keys or out-of-range array indices. - * @group variant_funcs - * @since 4.3.0 + * @param x + * the array column to be trimmed. A column that evaluates to an array. + * @param n + * the number of elements to remove from the end of the array. Must be between 0 and the + * number of elements in the array (inclusive). + * + * @group array_funcs + * @since 4.4.0 + * @return + * Returns a column that evaluates to an array. */ - def try_variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = - Column.fn("try_variant_set", v, lit(path), value, lit(createIfMissing)) + def trim_array(x: Column, n: Int): Column = trim_array(x, lit(n)) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of - * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + * Returns the given array `x` with the last `n` elements removed. Raises an error if `n` is + * negative or greater than the number of elements in the array. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the JSONPath string identifying the target array. A valid path should - * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. A column that evaluates to a string. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * @param x + * the array column to be trimmed. A column that evaluates to an array. + * @param n + * the number of elements to remove from the end of the array. Must be between 0 and the + * number of elements in the array (inclusive). + * + * @group array_funcs + * @since 4.4.0 + * @return + * Returns a column that evaluates to an array. */ - def variant_array_append(v: Column, path: Column, value: Column): Column = - Column.fn("variant_array_append", v, path, value) + def trim_array(x: Column, n: Column): Column = Column.fn("trim_array", x, n) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of - * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. - * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the JSONPath identifying the target array. A valid path should start with `$` and is - * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. A - * string. Must be a constant. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * Concatenates the elements of `column` using the `delimiter`. Null values are replaced with + * `nullReplacement`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. + * @param nullReplacement + * the string used to replace null values. A column that evaluates to a string. + * @group array_funcs + * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ - def variant_array_append(v: Column, path: String, value: Column): Column = - Column.fn("variant_array_append", v, lit(path), value) + def array_join(column: Column, delimiter: String, nullReplacement: String): Column = + Column.fn("array_join", column, lit(delimiter), lit(nullReplacement)) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an - * incompatible type, the target is not an array, or if any argument is NULL. - * - * @param v - * a variant column. - * @param path - * the column containing the JSONPath string identifying the target array. A valid path should - * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, - * or `["name"]`. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 + * Concatenates the elements of `column` using the `delimiter`. + * @param column + * the input column containing the array. A column that evaluates to an array. + * @param delimiter + * the string used to join the array elements. A column that evaluates to a string. + * @group array_funcs + * @since 2.4.0 + * @return + * Returns a column that evaluates to a string. */ - def try_variant_array_append(v: Column, path: Column, value: Column): Column = - Column.fn("try_variant_array_append", v, path, value) + def array_join(column: Column, delimiter: String): Column = + Column.fn("array_join", column, lit(delimiter)) /** - * Appends a value to the array in a variant at the given JSONPath location. Returns the variant - * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an - * incompatible type, the target is not an array, or if any argument is NULL. + * Concatenates multiple input columns together into a single column. The function works with + * strings, binary and compatible array columns. * - * @param v - * a variant column. - * @param path - * the JSONPath identifying the target array. A valid path should start with `$` and is - * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. - * @param value - * the value to append. Any expression castable to variant. - * @group variant_funcs - * @since 4.3.0 - */ - def try_variant_array_append(v: Column, path: String, value: Column): Column = - Column.fn("try_variant_array_append", v, lit(path), value) - - /** - * Recursively removes object fields and array elements whose value is a variant null. Returns - * NULL if `v` is NULL. + * @param exprs + * Input columns to concatenate. A column that evaluates to a string, binary or an array. + * @note + * Returns null if any of the input columns are null. * - * @param v - * a variant column. - * @group variant_funcs - * @since 4.3.0 + * @group collection_funcs + * @since 1.5.0 + * @return + * Returns a column of the same type as the input. */ - def variant_strip_nulls(v: Column): Column = Column.fn("variant_strip_nulls", v) + @scala.annotation.varargs + def concat(exprs: Column*): Column = Column.fn("concat", exprs: _*) /** - * Recursively removes object fields and array elements whose value is a variant null, unless - * `includeArrays` is false, in which case null array elements are kept. Returns NULL if any - * argument is NULL. + * Locates the position of the first occurrence of the value in the given array as long. Returns + * null if either of the arguments are null. * - * @param v - * a variant column. - * @param includeArrays - * whether null elements are also removed from arrays. - * @group variant_funcs - * @since 4.3.0 + * @param column + * The array to search. A column that evaluates to an array. + * @param value + * The value to locate. A column. + * @note + * The position is not zero based, but 1 based index. Returns 0 if value could not be found in + * array. + * + * @group array_funcs + * @since 2.4.0 + * @return + * Returns a column that evaluates to a long. */ - def variant_strip_nulls(v: Column, includeArrays: Boolean): Column = - Column.fn("variant_strip_nulls", v, lit(includeArrays)) + def array_position(column: Column, value: Any): Column = + Column.fn("array_position", column, lit(value)) /** - * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. + * Returns element of array at given index in value if column is array. Returns value for the + * given key in value if column is map. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the extraction path. A valid path should start with `$` and is followed by zero or more - * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. + * @group collection_funcs + * @since 2.4.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column of the element type of the input array, or the value type of the input + * map. */ - def variant_get(v: Column, path: String, targetType: String): Column = - Column.fn("variant_get", v, lit(path), lit(targetType)) + def element_at(column: Column, value: Any): Column = Column.fn("element_at", column, lit(value)) /** - * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. + * (array, index) - Returns element of array at given (1-based) index. If Index is 0, Spark will + * throw an error. If index < 0, accesses elements from the last to the first. The function + * always returns NULL if the index exceeds the length of the array. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the extraction path strings. A valid path string should start with - * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or - * `["name"]`. A column that evaluates to a string. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * (map, key) - Returns value for given key. The function always returns NULL if the key is not + * contained in the map. + * + * @param column + * The array or map to extract from. A column that evaluates to an array or a map. + * @param value + * The 1-based index for arrays, or the key for maps. A column. + * @group collection_funcs + * @since 3.5.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column of the element type of the input array, or the value type of the input + * map. */ - def variant_get(v: Column, path: Column, targetType: String): Column = - Column.fn("variant_get", v, path, lit(targetType)) + def try_element_at(column: Column, value: Column): Column = + Column.fn("try_element_at", column, value) /** - * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist or the cast fails.. + * Returns element of array at given (0-based) index. If the index points outside of the array + * boundaries, then this function returns NULL. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the extraction path. A valid path should start with `$` and is followed by zero or more - * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * @param column + * The array to extract from. A column that evaluates to an array. + * @param index + * The 0-based index. A column that evaluates to an integral. + * @group array_funcs + * @since 3.4.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column of the element type of the input array. */ - def try_variant_get(v: Column, path: String, targetType: String): Column = - Column.fn("try_variant_get", v, lit(path), lit(targetType)) + def get(column: Column, index: Column): Column = Column.fn("get", column, index) /** - * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to - * `targetType`. Returns null if the path does not exist or the cast fails.. + * Sorts the input array in ascending order. Null elements will be placed at the end of the + * returned array. NaN is greater than any non-NaN elements for double/float type. * - * @param v - * a variant column. A column that evaluates to a variant. - * @param path - * the column containing the extraction path strings. A valid path string should start with - * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or - * `["name"]`. A column that evaluates to a string. - * @param targetType - * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. - * @group variant_funcs - * @since 4.0.0 + * The elements of the input array must be orderable. For example, when the array elements are + * structs, the default comparator compares the struct fields in schema order. Therefore, all + * fields in the struct must be orderable. If the default comparator does not support the input + * type, you can specify a custom comparator. + * + * @param e + * The array to sort. A column that evaluates to an array. + * @group collection_funcs + * @since 2.4.0 * @return - * Returns a column of the type specified by the `targetType` argument. + * Returns a column that evaluates to an array. */ - def try_variant_get(v: Column, path: Column, targetType: String): Column = - Column.fn("try_variant_get", v, lit(path), lit(targetType)) + def array_sort(e: Column): Column = Column.fn("array_sort", e) /** - * Returns schema in the SQL format of a variant. + * Sorts the input array based on the given comparator function. The comparator will take two + * arguments representing two elements of the array. It returns a negative integer, 0, or a + * positive integer as the first element is less than, equal to, or greater than the second + * element. If the comparator function returns null, the function will fail and raise an error. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.0.0 + * @param e + * The array to sort. A column that evaluates to an array. + * @param comparator + * A binary comparator function that returns a negative integer, 0, or a positive integer as + * the first element is less than, equal to, or greater than the second element. + * @group collection_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) + def array_sort(e: Column, comparator: (Column, Column) => Column): Column = + Column.fn("array_sort", e, createLambda(comparator)) /** - * Returns the merged schema in the SQL format of a variant column. + * Remove all elements that equal to element from the given array. * - * @param v - * a variant column. A column that evaluates to a variant. - * @group variant_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a string. + * @param column + * The array to remove from. A column that evaluates to an array. + * @param element + * The element to remove. A column. + * @group array_funcs + * @since 2.4.0 + * @return + * Returns a column that evaluates to an array. */ - def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // XML Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def array_remove(column: Column, element: Any): Column = + Column.fn("array_remove", column, lit(element)) - // scalastyle:off line.size.limit /** - * Parses a column containing a XML string into the data type corresponding to the specified - * schema. Returns `null`, in the case of an unparseable string. + * Remove all null elements from the given array. * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a - * constant. - * @param options - * options to control how the XML is parsed. accepts the same options and the XML data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param column + * The array to compact. A column that evaluates to an array. + * @group array_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to an array. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = - from_xml(e, lit(schema.sql), options.asScala.iterator) + def array_compact(column: Column): Column = Column.fn("array_compact", column) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a XML string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * Returns an array containing value as well as all elements from array. The new element is + * positioned at the beginning of the array. * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. - * @param options - * options to control how the XML is parsed. accepts the same options and the xml data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param column + * The array to prepend to. A column that evaluates to an array. + * @param element + * The element to prepend. A column. + * @group array_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to an array. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { - from_xml(e, lit(schema), options) - } + def array_prepend(column: Column, element: Any): Column = + Column.fn("array_prepend", column, lit(element)) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a XML string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. - * + * Removes duplicate values from the array. * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A column that evaluates to a string. - * @group xml_funcs - * @since 4.0.0 + * The array to deduplicate. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to an array. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: Column): Column = { - from_xml(e, schema, Iterator.empty) - } + def array_distinct(e: Column): Column = Column.fn("array_distinct", e) - // scalastyle:off line.size.limit /** - * (Java-specific) Parses a column containing a XML string into a `StructType` with the - * specified schema. Returns `null`, in the case of an unparseable string. + * Returns an array of the elements in the intersection of the given two arrays, without + * duplicates. * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A column that evaluates to a string. - * @param options - * options to control how the XML is parsed. accepts the same options and the XML data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to an array. */ - // scalastyle:on line.size.limit - def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = - from_xml(e, schema, options.asScala.iterator) + def array_intersect(col1: Column, col2: Column): Column = + Column.fn("array_intersect", col1, col2) /** - * Parses a column containing a XML string into the data type corresponding to the specified - * schema. Returns `null`, in the case of an unparseable string. - * - * @param e - * a string column containing XML data. A column that evaluates to a string. - * @param schema - * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a - * constant. + * Adds an item into a given array at a specified position * - * @group xml_funcs - * @since 4.0.0 + * @param arr + * The array to insert into. A column that evaluates to an array. + * @param pos + * The 1-based position at which to insert (negative counts from the end). A column that + * evaluates to an integral. + * @param value + * The value to insert. A column. + * @group array_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a struct. + * Returns a column that evaluates to an array. */ - def from_xml(e: Column, schema: StructType): Column = - from_xml(e, schema, Map.empty[String, String].asJava) - - private def from_xml(e: Column, schema: Column, options: Iterator[(String, String)]): Column = { - Column.fnWithOptions("from_xml", options, e, schema) - } + def array_insert(arr: Column, pos: Column, value: Column): Column = + Column.fn("array_insert", arr, pos, value) /** - * Parses a XML string and infers its schema in DDL format. + * Returns an array of the elements in the union of the given two arrays, without duplicates. * - * @param xml - * a XML string. A string. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) + def array_union(col1: Column, col2: Column): Column = + Column.fn("array_union", col1, col2) /** - * Parses a XML string and infers its schema in DDL format. + * Returns an array of the elements in the first array but not in the second array, without + * duplicates. The order of elements in the result is not determined * - * @param xml - * a foldable string column containing a XML string. A column that evaluates to a string. - * @group xml_funcs - * @since 4.0.0 + * @param col1 + * The first array. A column that evaluates to an array. + * @param col2 + * The second array. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) + def array_except(col1: Column, col2: Column): Column = + Column.fn("array_except", col1, col2) - // scalastyle:off line.size.limit + private def createLambda(f: Column => Column) = { + val x = internal.UnresolvedNamedLambdaVariable("x") + val function = f(Column(x)).node + Column(internal.LambdaFunction(function, Seq(x))) + } - /** - * Parses a XML string and infers its schema in DDL format using options. - * - * @param xml - * a foldable string column containing XML data. A column that evaluates to a string. - * @param options - * options to control how the xml is parsed. accepts the same options and the XML data source. - * See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @return - * a column with string literal containing schema in DDL format. Returns a column that - * evaluates to a string. - * @group xml_funcs - * @since 4.0.0 - */ - // scalastyle:on line.size.limit - def schema_of_xml(xml: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("schema_of_xml", options.asScala.iterator, xml) + private def createLambda(f: (Column, Column) => Column) = { + val x = internal.UnresolvedNamedLambdaVariable("x") + val y = internal.UnresolvedNamedLambdaVariable("y") + val function = f(Column(x), Column(y)).node + Column(internal.LambdaFunction(function, Seq(x, y))) + } - // scalastyle:off line.size.limit + private def createLambda(f: (Column, Column, Column) => Column) = { + val x = internal.UnresolvedNamedLambdaVariable("x") + val y = internal.UnresolvedNamedLambdaVariable("y") + val z = internal.UnresolvedNamedLambdaVariable("z") + val function = f(Column(x), Column(y), Column(z)).node + Column(internal.LambdaFunction(function, Seq(x, y, z))) + } /** - * (Java-specific) Converts a column containing a `StructType` into a XML string with the - * specified schema. Throws an exception, in the case of an unsupported type. + * Returns an array of elements after applying a transformation to each element in the input + * array. + * {{{ + * df.select(transform(col("i"), x => x + 1)) + * }}} * - * @param e - * a column containing a struct. A column that evaluates to a string. - * @param options - * options to control how the struct column is converted into a XML string. It accepts the - * same options as the XML data source. See Data - * Source Option in the version you use. A map of string options. Must be a constant. - * @group xml_funcs - * @since 4.0.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => transformed_col, the lambda function to transform the input column. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - // scalastyle:on line.size.limit - def to_xml(e: Column, options: java.util.Map[String, String]): Column = - Column.fnWithOptions("to_xml", options.asScala.iterator, e) + def transform(column: Column, f: Column => Column): Column = + Column.fn("transform", column, createLambda(f)) /** - * Converts a column containing a `StructType` into a XML string with the specified schema. - * Throws an exception, in the case of an unsupported type. + * Returns an array of elements after applying a transformation to each element in the input + * array. + * {{{ + * df.select(transform(col("i"), (x, i) => x + i)) + * }}} * - * @param e - * a column containing a struct. A column that evaluates to a string. - * @group xml_funcs - * @since 4.0.0 - * @return - * Returns a column that evaluates to a string. - */ - def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) - - /** - * Returns a string array of values within the nodes of xml that match the XPath expression. + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * (col, index) => transformed_col, the lambda function to transform the input column given + * the index. Indices start at 0. * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @group collection_funcs + * @since 3.0.0 * @return * Returns a column that evaluates to an array. */ - def xpath(xml: Column, path: Column): Column = - Column.fn("xpath", xml, path) + def transform(column: Column, f: (Column, Column) => Column): Column = + Column.fn("transform", column, createLambda(f)) /** - * Returns true if the XPath expression evaluates to true, or if a matching node is found. + * Returns whether a predicate holds for one or more elements in the array. + * {{{ + * df.select(exists(col("i"), _ % 2 === 0)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to check the input column. + * + * @group collection_funcs + * @since 3.0.0 * @return * Returns a column that evaluates to a boolean. */ - def xpath_boolean(xml: Column, path: Column): Column = - Column.fn("xpath_boolean", xml, path) + def exists(column: Column, f: Column => Column): Column = + Column.fn("exists", column, createLambda(f)) /** - * Returns a double value, the value zero if no match is found, or NaN if a match is found but - * the value is non-numeric. + * Returns whether a predicate holds for every element in the array. + * {{{ + * df.select(forall(col("i"), x => x % 2 === 0)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to check the input column. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a boolean. */ - def xpath_double(xml: Column, path: Column): Column = - Column.fn("xpath_double", xml, path) + def forall(column: Column, f: Column => Column): Column = + Column.fn("forall", column, createLambda(f)) /** - * Returns a double value, the value zero if no match is found, or NaN if a match is found but - * the value is non-numeric. + * Returns an array of elements for which a predicate holds in a given array. + * {{{ + * df.select(filter(col("s"), x => x % 2 === 0)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * col => predicate, the Boolean predicate to filter the input column. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def xpath_number(xml: Column, path: Column): Column = - Column.fn("xpath_number", xml, path) + def filter(column: Column, f: Column => Column): Column = + Column.fn("filter", column, createLambda(f)) /** - * Returns a float value, the value zero if no match is found, or NaN if a match is found but - * the value is non-numeric. + * Returns an array of elements for which a predicate holds in a given array. + * {{{ + * df.select(filter(col("s"), (x, i) => i % 2 === 0)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param column + * the input array column. A column that evaluates to an array. + * @param f + * (col, index) => predicate, the Boolean predicate to filter the input column given the + * index. Indices start at 0. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a float. + * Returns a column that evaluates to an array. */ - def xpath_float(xml: Column, path: Column): Column = - Column.fn("xpath_float", xml, path) + def filter(column: Column, f: (Column, Column) => Column): Column = + Column.fn("filter", column, createLambda(f)) /** - * Returns an integer value, or the value zero if no match is found, or a match is found but the - * value is non-numeric. + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. The final state is converted into the final result by applying a finish + * function. + * {{{ + * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value. + * @param finish + * combined_value => final_value, the lambda function to convert the combined value of all + * inputs to final result. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column of the same type as the initial value. */ - def xpath_int(xml: Column, path: Column): Column = - Column.fn("xpath_int", xml, path) + def aggregate( + expr: Column, + initialValue: Column, + merge: (Column, Column) => Column, + finish: Column => Column): Column = + Column.fn("aggregate", expr, initialValue, createLambda(merge), createLambda(finish)) /** - * Returns a long integer value, or the value zero if no match is found, or a match is found but - * the value is non-numeric. + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. + * {{{ + * df.select(aggregate(col("i"), lit(0), (acc, x) => acc + x)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs - * @since 3.5.0 + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the same type as the initial value. */ - def xpath_long(xml: Column, path: Column): Column = - Column.fn("xpath_long", xml, path) + def aggregate(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = + aggregate(expr, initialValue, merge, c => c) /** - * Returns a short integer value, or the value zero if no match is found, or a match is found - * but the value is non-numeric. + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. The final state is converted into the final result by applying a finish + * function. + * {{{ + * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x, _ * 10)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value. + * @param finish + * combined_value => final_value, the lambda function to convert the combined value of all + * inputs to final result. + * + * @group collection_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a short. + * Returns a column of the same type as the initial value. */ - def xpath_short(xml: Column, path: Column): Column = - Column.fn("xpath_short", xml, path) + def reduce( + expr: Column, + initialValue: Column, + merge: (Column, Column) => Column, + finish: Column => Column): Column = + Column.fn("reduce", expr, initialValue, createLambda(merge), createLambda(finish)) /** - * Returns the text contents of the first xml node that matches the XPath expression. + * Applies a binary operator to an initial state and all elements in the array, and reduces this + * to a single state. + * {{{ + * df.select(reduce(col("i"), lit(0), (acc, x) => acc + x)) + * }}} * - * @param xml - * the XML column to evaluate. A column that evaluates to a string. - * @param path - * the XPath expression to match. A column that evaluates to a string. Must be a constant. - * @group xml_funcs + * @param expr + * the input array column. A column that evaluates to an array. + * @param initialValue + * the initial value. A column of any type. + * @param merge + * (combined_value, input_value) => combined_value, the merge function to merge an input value + * to the combined_value + * @group collection_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the same type as the initial value. */ - def xpath_string(xml: Column, path: Column): Column = - Column.fn("xpath_string", xml, path) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // URL Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def reduce(expr: Column, initialValue: Column, merge: (Column, Column) => Column): Column = + reduce(expr, initialValue, merge, c => c) /** - * Extracts a part from a URL. + * Merge two given arrays, element-wise, into a single array using a function. If one array is + * shorter, nulls are appended at the end to match the length of the longer array, before + * applying the function. + * {{{ + * df.select(zip_with(df1("val1"), df1("val2"), (x, y) => x + y)) + * }}} * - * @param url - * A column of strings, each representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @param key - * The key of a query parameter in the URL. A column that evaluates to a string. - * @group url_funcs - * @since 4.0.0 + * @param left + * the left input array column. A column that evaluates to an array. + * @param right + * the right input array column. A column that evaluates to an array. + * @param f + * (lCol, rCol) => col, the lambda function to merge two input columns into one column. + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an array. */ - def try_parse_url(url: Column, partToExtract: Column, key: Column): Column = - Column.fn("try_parse_url", url, partToExtract, key) + def zip_with(left: Column, right: Column, f: (Column, Column) => Column): Column = + Column.fn("zip_with", left, right, createLambda(f)) /** - * Extracts a part from a URL. + * Applies a function to every key-value pair in a map and returns a map with the results of + * those applications as the new keys for the pairs. + * {{{ + * df.select(transform_keys(col("i"), (k, v) => k + v)) + * }}} * - * @param url - * A column of strings, each representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @group url_funcs - * @since 4.0.0 + * @param expr + * the input map column. A column that evaluates to a map. + * @param f + * (key, value) => new_key, the lambda function to transform the key of input map column + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - def try_parse_url(url: Column, partToExtract: Column): Column = - Column.fn("try_parse_url", url, partToExtract) + def transform_keys(expr: Column, f: (Column, Column) => Column): Column = + Column.fn("transform_keys", expr, createLambda(f)) /** - * Extracts a part from a URL. + * Applies a function to every key-value pair in a map and returns a map with the results of + * those applications as the new values for the pairs. + * {{{ + * df.select(transform_values(col("i"), (k, v) => k + v)) + * }}} * - * @param url - * A column of strings, each representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @param key - * The key of a query parameter in the URL. A column that evaluates to a string. - * @group url_funcs - * @since 3.5.0 + * @param expr + * the input map column. A column that evaluates to a map. + * @param f + * (key, value) => new_value, the lambda function to transform the value of input map column + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - def parse_url(url: Column, partToExtract: Column, key: Column): Column = - Column.fn("parse_url", url, partToExtract, key) + def transform_values(expr: Column, f: (Column, Column) => Column): Column = + Column.fn("transform_values", expr, createLambda(f)) /** - * Extracts a part from a URL. + * Returns a map whose key-value pairs satisfy a predicate. + * {{{ + * df.select(map_filter(col("m"), (k, v) => k * 10 === v)) + * }}} * - * @param url - * A column representing a URL. A column that evaluates to a string. - * @param partToExtract - * The part to extract from the URL. A column that evaluates to a string. - * @group url_funcs - * @since 3.5.0 + * @param expr + * the input map column. A column that evaluates to a map. + * @param f + * (key, value) => predicate, the Boolean predicate to filter the input map column + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - def parse_url(url: Column, partToExtract: Column): Column = - Column.fn("parse_url", url, partToExtract) + def map_filter(expr: Column, f: (Column, Column) => Column): Column = + Column.fn("map_filter", expr, createLambda(f)) /** - * Decodes a `str` in 'application/x-www-form-urlencoded' format using a specific encoding - * scheme. + * Merge two given maps, key-wise into a single map using a function. + * {{{ + * df.select(map_zip_with(df("m1"), df("m2"), (k, v1, v2) => k === v1 + v2)) + * }}} * - * @param str - * A URL-encoded string. A column that evaluates to a string. - * @group url_funcs - * @since 3.5.0 + * @param left + * the left input map column. A column that evaluates to a map. + * @param right + * the right input map column. A column that evaluates to a map. + * @param f + * (key, value1, value2) => new_value, the lambda function to merge the map values + * + * @group collection_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a map. */ - def url_decode(str: Column): Column = Column.fn("url_decode", str) + def map_zip_with(left: Column, right: Column, f: (Column, Column, Column) => Column): Column = + Column.fn("map_zip_with", left, right, createLambda(f)) /** - * This is a special version of `url_decode` that performs the same operation, but returns a - * NULL value instead of raising an error if the decoding cannot be performed. + * Creates a new row for each element in the given array or map column. Uses the default column + * name `col` for elements in the array and `key` and `value` for elements in the map unless + * specified otherwise. * - * @param str - * A URL-encoded string. A column that evaluates to a string. - * @group url_funcs - * @since 4.0.0 + * @param e + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 1.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ - def try_url_decode(str: Column): Column = Column.fn("try_url_decode", str) + def explode(e: Column): Column = Column.fn("explode", e) /** - * Translates a string into 'application/x-www-form-urlencoded' format using a specific encoding - * scheme. + * Creates a new row for each element in the given array or map column. Uses the default column + * name `col` for elements in the array and `key` and `value` for elements in the map unless + * specified otherwise. Unlike explode, if the array/map is null or empty then null is produced. * - * @param str - * A string to encode. A column that evaluates to a string. - * @group url_funcs - * @since 3.5.0 + * @param e + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 2.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the element type of the input array, or the key and value columns of + * the input map. */ - def url_encode(str: Column): Column = Column.fn("url_encode", str) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Misc Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def explode_outer(e: Column): Column = Column.fn("explode_outer", e) /** - * Creates a string column for the file name of the current Spark task. + * Creates a new row for each element with position in the given array or map column. Uses the + * default column name `pos` for position, and `col` for elements in the array and `key` and + * `value` for elements in the map unless specified otherwise. * - * @group misc_funcs - * @since 1.6.0 + * @param e + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a string. + * Returns the position column and a column of the element type of the input array, or the + * position column and the key and value columns of the input map. */ - def input_file_name(): Column = Column.fn("input_file_name") + def posexplode(e: Column): Column = Column.fn("posexplode", e) /** - * A column expression that generates monotonically increasing 64-bit integers. - * - * The generated ID is guaranteed to be monotonically increasing and unique, but not - * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the - * record number within each partition in the lower 33 bits. The assumption is that the data - * frame has less than 1 billion partitions, and each partition has less than 8 billion records. - * - * As an example, consider a `DataFrame` with two partitions, each with 3 records. This - * expression would return the following IDs: - * - * {{{ - * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. - * }}} + * Creates a new row for each element with position in the given array or map column. Uses the + * default column name `pos` for position, and `col` for elements in the array and `key` and + * `value` for elements in the map unless specified otherwise. Unlike posexplode, if the + * array/map is null or empty then the row (null, null) is produced. * - * @group misc_funcs - * @since 1.4.0 + * @param e + * the target column to explode. A column that evaluates to an array or a map. + * @group generator_funcs + * @since 2.2.0 * @return - * Returns a column that evaluates to a long. + * Returns the position column and a column of the element type of the input array, or the + * position column and the key and value columns of the input map. */ - @deprecated("Use monotonically_increasing_id()", "2.0.0") - def monotonicallyIncreasingId(): Column = monotonically_increasing_id() + def posexplode_outer(e: Column): Column = Column.fn("posexplode_outer", e) /** - * A column expression that generates monotonically increasing 64-bit integers. - * - * The generated ID is guaranteed to be monotonically increasing and unique, but not - * consecutive. The current implementation puts the partition ID in the upper 31 bits, and the - * record number within each partition in the lower 33 bits. The assumption is that the data - * frame has less than 1 billion partitions, and each partition has less than 8 billion records. - * - * As an example, consider a `DataFrame` with two partitions, each with 3 records. This - * expression would return the following IDs: - * - * {{{ - * 0, 1, 2, 8589934592 (1L << 33), 8589934593, 8589934594. - * }}} + * Creates a new row for each element in the given array of structs. * - * @group misc_funcs - * @since 1.6.0 + * @param e + * the target column to explode. A column that evaluates to an array of structs. + * @group generator_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a struct. */ - def monotonically_increasing_id(): Column = Column.fn("monotonically_increasing_id") + def inline(e: Column): Column = Column.fn("inline", e) /** - * Partition ID. - * - * @note - * This is non-deterministic because it depends on data partitioning and task scheduling. + * Creates a new row for each element in the given array of structs. Unlike inline, if the array + * is null or empty then null is produced for each nested column. * - * @group misc_funcs - * @since 1.6.0 + * @param e + * the target column to explode. A column that evaluates to an array of structs. + * @group generator_funcs + * @since 3.4.0 * @return - * Returns a column that evaluates to an integer. + * Returns a column that evaluates to a struct. */ - def spark_partition_id(): Column = Column.fn("spark_partition_id") + def inline_outer(e: Column): Column = Column.fn("inline_outer", e) /** - * Returns the current catalog. + * Extracts json object from a json string based on json path specified, and returns json string + * of the extracted json object. It will return null if the input json string is invalid. * - * @group misc_funcs - * @since 3.5.0 + * @param e + * the JSON string column. A column that evaluates to a string. + * @param path + * the JSON path to extract. A column that evaluates to a string. Must be a constant. + * @group json_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a string. */ - def current_catalog(): Column = Column.fn("current_catalog") + def get_json_object(e: Column, path: String): Column = + Column.fn("get_json_object", e, lit(path)) /** - * Returns the current database. + * Creates a new row for a json column according to the given field names. * - * @group misc_funcs - * @since 3.5.0 + * @param json + * the JSON string column. A column that evaluates to a string. + * @param fields + * the field names to extract. A column that evaluates to a string. Must be a constant. + * @group json_funcs + * @since 1.6.0 * @return * Returns a column that evaluates to a string. */ - def current_database(): Column = Column.fn("current_database") + @scala.annotation.varargs + def json_tuple(json: Column, fields: String*): Column = { + require(fields.nonEmpty, "at least 1 field name should be given.") + Column.fn("json_tuple", json +: fields.map(lit): _*) + } + // scalastyle:off line.size.limit /** - * Returns the current schema. + * (Scala-specific) Parses a column containing a JSON string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @group misc_funcs - * @since 3.5.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. Accepts the same options as the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a struct. */ - def current_schema(): Column = Column.fn("current_schema") + // scalastyle:on line.size.limit + def from_json(e: Column, schema: StructType, options: Map[String, String]): Column = + from_json(e, schema.asInstanceOf[DataType], options) + // scalastyle:off line.size.limit /** - * Returns the current SQL path as a comma-separated list of qualified schema names. + * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * - * @group misc_funcs - * @since 4.2.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.2.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def current_path(): Column = Column.fn("current_path") + // scalastyle:on line.size.limit + def from_json(e: Column, schema: DataType, options: Map[String, String]): Column = { + from_json(e, lit(schema.sql), options.iterator) + } + // scalastyle:off line.size.limit /** - * Returns the user name of current execution context. + * (Java-specific) Parses a column containing a JSON string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @group misc_funcs - * @since 3.5.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a struct. */ - def current_user(): Column = Column.fn("current_user") + // scalastyle:on line.size.limit + def from_json(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = + from_json(e, schema, options.asScala.toMap) + // scalastyle:off line.size.limit /** - * Returns null if the condition is true, and throws an exception otherwise. + * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * - * @param c - * The condition to check. A column that evaluates to a boolean. - * @group misc_funcs - * @since 3.1.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.2.0 * @return - * Returns a column that always evaluates to NULL. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def assert_true(c: Column): Column = Column.fn("assert_true", c) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: DataType, options: java.util.Map[String, String]): Column = { + from_json(e, schema, options.asScala.toMap) + } /** - * Returns null if the condition is true; throws an exception with the error message otherwise. + * Parses a column containing a JSON string into a `StructType` with the specified schema. + * Returns `null`, in the case of an unparseable string. * - * @param c - * The condition to check. A column that evaluates to a boolean. * @param e - * The error message to throw. A column that evaluates to a string. - * @group misc_funcs - * @since 3.1.0 + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that always evaluates to NULL. + * Returns a column that evaluates to a struct. */ - def assert_true(c: Column, e: Column): Column = Column.fn("assert_true", c, e) + def from_json(e: Column, schema: StructType): Column = + from_json(e, schema, Map.empty[String, String]) /** - * Throws an exception with the provided error message. + * Parses a column containing a JSON string into a `MapType` with `StringType` as keys type, + * `StructType` or `ArrayType` with the specified schema. Returns `null`, in the case of an + * unparseable string. * - * @param c - * The error message to throw. A column that evaluates to a string. - * @group misc_funcs - * @since 3.1.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A string, StructType or DataType. Must be a + * constant. + * + * @group json_funcs + * @since 2.2.0 * @return - * Returns a column that always evaluates to NULL. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def raise_error(c: Column): Column = Column.fn("raise_error", c) + def from_json(e: Column, schema: DataType): Column = + from_json(e, schema, Map.empty[String, String]) + // scalastyle:off line.size.limit /** - * Returns the user name of current execution context. + * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * - * @group misc_funcs - * @since 3.5.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def user(): Column = Column.fn("user") + // scalastyle:on line.size.limit + def from_json(e: Column, schema: String, options: java.util.Map[String, String]): Column = { + from_json(e, schema, options.asScala.toMap) + } + // scalastyle:off line.size.limit /** - * Returns the user name of current execution context. + * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` with the specified schema. Returns `null`, in the + * case of an unparseable string. * - * @group misc_funcs - * @since 4.0.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def session_user(): Column = Column.fn("session_user") + // scalastyle:on line.size.limit + def from_json(e: Column, schema: String, options: Map[String, String]): Column = { + from_json(e, lit(schema), options.asJava) + } /** - * Returns an universally unique identifier (UUID) string. The value is returned as a canonical - * UUID 36-character string. + * (Scala-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns + * `null`, in the case of an unparseable string. * - * @group misc_funcs - * @since 3.5.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A column that evaluates to a string. + * + * @group json_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def uuid(): Column = Column.fn("uuid", lit(SparkClassUtils.random.nextLong)) + def from_json(e: Column, schema: Column): Column = { + from_json(e, schema, Map.empty[String, String].asJava) + } + // scalastyle:off line.size.limit /** - * Returns an universally unique identifier (UUID) string. The value is returned as a canonical - * UUID 36-character string. + * (Java-specific) Parses a column containing a JSON string into a `MapType` with `StringType` + * as keys type, `StructType` or `ArrayType` of `StructType`s with the specified schema. Returns + * `null`, in the case of an unparseable string. * - * @param seed - * The random number seed to use. A column that evaluates to an integral. Must be a constant. - * @group misc_funcs - * @since 4.1.0 + * @param e + * a string column containing JSON data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the json string. A column that evaluates to a string. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a string. + * Returns a column of the type given by the schema (a struct, array, or map). */ - def uuid(seed: Column): Column = Column.fn("uuid", seed) + // scalastyle:on line.size.limit + def from_json(e: Column, schema: Column, options: java.util.Map[String, String]): Column = { + from_json(e, schema, options.asScala.iterator) + } + + private def from_json( + e: Column, + schema: Column, + options: Iterator[(String, String)]): Column = { + Column.fnWithOptions("from_json", options, e, schema) + } /** - * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and the - * given hash `algorithm`. The result is returned as raw MAC bytes; wrap it with `hex` or - * `base64` for a textual value. + * Parses a JSON string and constructs a Variant value. Returns null if the input string is not + * a valid JSON value. * - * @param key - * The secret key, as a binary value. - * @param message - * The message to authenticate, as a binary value. - * @param algorithm - * The hash algorithm. Valid values: SHA-224, SHA-256, SHA-384, SHA-512, SHA-1, MD5. + * @param json + * a string column that contains JSON data. A column that evaluates to a string. * - * @group misc_funcs - * @since 4.3.0 + * @group variant_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ - def hmac(key: Column, message: Column, algorithm: Column): Column = - Column.fn("hmac", key, message, algorithm) + def try_parse_json(json: Column): Column = Column.fn("try_parse_json", json) /** - * Returns the keyed-hash message authentication code (HMAC) of `message` using `key` and - * SHA-256. The result is returned as raw MAC bytes; wrap it with `hex` or `base64` for a - * textual value. To use a different algorithm, call the three-argument overload. - * - * @param key - * The secret key, as a binary value. - * @param message - * The message to authenticate, as a binary value. + * Parses a JSON string and constructs a Variant value. * - * @group misc_funcs - * @since 4.3.0 + * @param json + * a string column that contains JSON data. A column that evaluates to a string. + * @group variant_funcs + * @since 4.0.0 + * @return + * Returns a column that evaluates to a variant. */ - def hmac(key: Column, message: Column): Column = - Column.fn("hmac", key, message) + def parse_json(json: Column): Column = Column.fn("parse_json", json) /** - * Returns an encrypted value of `input` using AES in given `mode` with the specified `padding`. - * Key lengths of 16, 24 and 32 bits are supported. Supported combinations of (`mode`, - * `padding`) are ('ECB', 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional initialization - * vectors (IVs) are only supported for CBC and GCM modes. These must be 16 bytes for CBC and 12 - * bytes for GCM. If not provided, a random vector will be generated and prepended to the - * output. Optional additional authenticated data (AAD) is only supported for GCM. If provided - * for encryption, the identical AAD value must be provided for decryption. The default mode is - * GCM. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param iv - * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a - * binary. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. A column that - * evaluates to a binary. + * Converts a column containing nested inputs (array/map/struct) into a variants where maps and + * structs are converted to variant objects which are unordered unlike SQL structs. Input maps + * can only have string keys. * - * @group misc_funcs - * @since 3.5.0 + * @param col + * a column with a nested schema or column name. A column that evaluates to a struct, array, + * map, or variant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a variant. */ - def aes_encrypt( - input: Column, - key: Column, - mode: Column, - padding: Column, - iv: Column, - aad: Column): Column = Column.fn("aes_encrypt", input, key, mode, padding, iv, aad) + def to_variant_object(col: Column): Column = Column.fn("to_variant_object", col) /** - * Returns an encrypted value of `input`. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param iv - * Optional initialization vector. Only supported for CBC and GCM modes. Valid values: None or - * "". 16-byte array for CBC mode. 12-byte array for GCM mode. A column that evaluates to a - * binary. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * Creates a variant object from the given arrays of keys and values. The keys must be non-null + * strings and the two arrays must have the same length. * - * @group misc_funcs - * @since 3.5.0 + * @param keys + * a column that evaluates to an array of string keys. + * @param values + * a column that evaluates to an array of values. + * @group variant_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a variant. */ - def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column, iv: Column): Column = - Column.fn("aes_encrypt", input, key, mode, padding, iv) + def variant_from_arrays(keys: Column, values: Column): Column = + Column.fn("variant_from_arrays", keys, values) /** - * Returns an encrypted value of `input`. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * Creates a variant object from an array of key/value struct entries. The keys must be non-null + * strings. * - * @group misc_funcs - * @since 3.5.0 + * @param entries + * a column that evaluates to an array of key/value structs. + * @group variant_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a variant. */ - def aes_encrypt(input: Column, key: Column, mode: Column, padding: Column): Column = - Column.fn("aes_encrypt", input, key, mode, padding) + def variant_from_entries(entries: Column): Column = + Column.fn("variant_from_entries", entries) /** - * Returns an encrypted value of `input`. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to encrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * Check if a variant value is a variant null. Returns true if and only if the input is a + * variant null and false otherwise (including in the case of SQL NULL). * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def aes_encrypt(input: Column, key: Column, mode: Column): Column = - Column.fn("aes_encrypt", input, key, mode) + def is_variant_null(v: Column): Column = Column.fn("is_variant_null", v) /** - * Returns an encrypted value of `input`. - * - * @param input - * The binary value to encrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to encrypt the data. A column that evaluates to a binary. - * @see - * `org.apache.spark.sql.functions.aes_encrypt(Column, Column, Column, Column, Column, - * Column)` + * Check if a variant value is valid. Returns true if the variant is valid, false if it is + * malformed, and NULL if the input is NULL. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.2.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def aes_encrypt(input: Column, key: Column): Column = - Column.fn("aes_encrypt", input, key) + def is_valid_variant(v: Column): Column = Column.fn("is_valid_variant", v) /** - * Returns a decrypted value of `input` using AES in `mode` with `padding`. Key lengths of 16, - * 24 and 32 bits are supported. Supported combinations of (`mode`, `padding`) are ('ECB', - * 'PKCS'), ('GCM', 'NONE') and ('CBC', 'PKCS'). Optional additional authenticated data (AAD) is - * only supported for GCM. If provided for encryption, the identical AAD value must be provided - * for decryption. The default mode is GCM. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. A column that - * evaluates to a binary. + * Removes fields or array elements from a variant at the given JSONPath locations. Multiple + * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the first JSONPath string. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A column that evaluates to a string. + * @param paths + * additional JSONPath arguments, applied after `path` in order. A column that evaluates to a + * string. + * @group variant_funcs + * @since 5.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a variant. */ - def aes_decrypt( - input: Column, - key: Column, - mode: Column, - padding: Column, - aad: Column): Column = - Column.fn("aes_decrypt", input, key, mode, padding, aad) + @scala.annotation.varargs + def variant_delete(v: Column, path: Column, paths: Column*): Column = + Column.fn("variant_delete", (v +: path +: paths): _*) /** - * Returns a decrypted value of `input`. + * Removes fields or array elements from a variant at the given JSONPath locations. Multiple + * paths are applied left to right. Returns NULL if `v` is NULL; NULL paths are skipped. * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the first JSONPath identifying a deletion target. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A string. Must be a constant. + * @param paths + * additional JSONPath strings, applied after `path` in order. A string. Must be a constant. + * @group variant_funcs + * @since 5.0.0 + * @return + * Returns a column that evaluates to a variant. + */ + @scala.annotation.varargs + def variant_delete(v: Column, path: String, paths: String*): Column = + Column.fn("variant_delete", (v +: lit(path) +: paths.map(lit)): _*) + + /** + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field (error if it already exists); an array path inserts at the index, shifting later + * elements right. Missing intermediate keys are created. Throws an error if a path segment hits + * a value of an incompatible type. Returns NULL if any argument is NULL. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the insertion target. A valid path + * should start with `$` and is followed by one or more segments like `[123]`, `.name`, + * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a + * string. + * @param value + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a variant. + */ + def variant_insert(v: Column, path: Column, value: Column): Column = + Column.fn("variant_insert", v, path, value) + + /** + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field (error if it already exists); an array path inserts at the index, shifting later + * elements right. Missing intermediate keys are created. Throws an error if a path segment hits + * a value of an incompatible type. Returns NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the insertion target. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A string. Must be a constant. + * @param value + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a variant. + */ + def variant_insert(v: Column, path: String, value: Column): Column = + Column.fn("variant_insert", v, lit(path), value) + + /** + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field; an array path inserts at the index, shifting later elements right. Missing + * intermediate keys are created. Returns NULL if the field already exists or a path segment + * hits a value of an incompatible type, or if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the insertion target. A valid path + * should start with `$` and is followed by one or more segments like `[123]`, `.name`, + * `['name']`, or `["name"]`. The root path `$` is not allowed. A column that evaluates to a + * string. + * @param value + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + */ + def try_variant_insert(v: Column, path: Column, value: Column): Column = + Column.fn("try_variant_insert", v, path, value) + + /** + * Inserts a value into a variant at the given JSONPath location. An object path adds a new + * field; an array path inserts at the index, shifting later elements right. Missing + * intermediate keys are created. Returns NULL if the field already exists or a path segment + * hits a value of an incompatible type, or if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the insertion target. A valid path should start with `$` and is + * followed by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root + * path `$` is not allowed. A string. Must be a constant. + * @param value + * the value to insert. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + */ + def try_variant_insert(v: Column, path: String, value: Column): Column = + Column.fn("try_variant_insert", v, lit(path), value) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Throws an error if a path segment hits a value of an incompatible type. + * Returns NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. + * @param value + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + */ + def variant_set(v: Column, path: Column, value: Column): Column = + Column.fn("variant_set", v, path, value) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Throws an error if a path segment hits a value of an incompatible type. + * Returns NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. A string. Must be a constant. + * @param value + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + */ + def variant_set(v: Column, path: String, value: Column): Column = + Column.fn("variant_set", v, lit(path), value) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns + * NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. A column that evaluates to a string. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. A boolean. Must be a + * constant. + * @group variant_funcs + * @since 4.3.0 + */ + def variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = + Column.fn("variant_set", v, path, value, lit(createIfMissing)) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Throws an error if a path segment hits a value of an incompatible type. Returns + * NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. A string. Must be a constant. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. A boolean. Must be a + * constant. + * @group variant_funcs + * @since 4.3.0 + */ + def variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = + Column.fn("variant_set", v, lit(path), value, lit(createIfMissing)) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if + * any argument is NULL. + * + * @param v + * a variant column. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. + * @param value + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + */ + def try_variant_set(v: Column, path: Column, value: Column): Column = + Column.fn("try_variant_set", v, path, value) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created. Returns NULL if a path segment hits a value of an incompatible type, or if + * any argument is NULL. + * + * @param v + * a variant column. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. + * @param value + * the value to set. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + */ + def try_variant_set(v: Column, path: String, value: Column): Column = + Column.fn("try_variant_set", v, lit(path), value) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any + * argument is NULL. + * + * @param v + * a variant column. + * @param path + * the column containing the JSONPath string identifying the set target. A valid path should + * start with `$` and is followed by one or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. The root path `$` is not allowed. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. + * @group variant_funcs + * @since 4.3.0 + */ + def try_variant_set(v: Column, path: Column, value: Column, createIfMissing: Boolean): Column = + Column.fn("try_variant_set", v, path, value, lit(createIfMissing)) + + /** + * Sets or upserts a value in a variant at the given JSONPath location. An existing object field + * or array element at the target is replaced. A missing field, array index, or intermediate + * path is created, unless `createIfMissing` is false, in which case the variant is left + * unchanged. Returns NULL if a path segment hits a value of an incompatible type, or if any + * argument is NULL. + * + * @param v + * a variant column. + * @param path + * the JSONPath identifying the set target. A valid path should start with `$` and is followed + * by one or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. The root path `$` + * is not allowed. + * @param value + * the value to set. Any expression castable to variant. + * @param createIfMissing + * whether to create missing keys or out-of-range array indices. + * @group variant_funcs + * @since 4.3.0 + */ + def try_variant_set(v: Column, path: String, value: Column, createIfMissing: Boolean): Column = + Column.fn("try_variant_set", v, lit(path), value, lit(createIfMissing)) + + /** + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of + * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the JSONPath string identifying the target array. A valid path should + * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. A column that evaluates to a string. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 + */ + def variant_array_append(v: Column, path: Column, value: Column): Column = + Column.fn("variant_array_append", v, path, value) + + /** + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Throws an error if a path segment hits a value of + * an incompatible type or the target is not an array. Returns NULL if any argument is NULL. + * + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the JSONPath identifying the target array. A valid path should start with `$` and is + * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. A + * string. Must be a constant. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = - Column.fn("aes_decrypt", input, key, mode, padding) + def variant_array_append(v: Column, path: String, value: Column): Column = + Column.fn("variant_array_append", v, lit(path), value) /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an + * incompatible type, the target is not an array, or if any argument is NULL. * - * @group misc_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a binary. + * @param v + * a variant column. + * @param path + * the column containing the JSONPath string identifying the target array. A valid path should + * start with `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, + * or `["name"]`. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def aes_decrypt(input: Column, key: Column, mode: Column): Column = - Column.fn("aes_decrypt", input, key, mode) + def try_variant_array_append(v: Column, path: Column, value: Column): Column = + Column.fn("try_variant_array_append", v, path, value) /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @see - * `org.apache.spark.sql.functions.aes_decrypt(Column, Column, Column, Column, Column)` + * Appends a value to the array in a variant at the given JSONPath location. Returns the variant + * unchanged if a path key or index is absent. Returns NULL if a path segment hits a value of an + * incompatible type, the target is not an array, or if any argument is NULL. * - * @group misc_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a binary. + * @param v + * a variant column. + * @param path + * the JSONPath identifying the target array. A valid path should start with `$` and is + * followed by zero or more segments like `[123]`, `.name`, `['name']`, or `["name"]`. + * @param value + * the value to append. Any expression castable to variant. + * @group variant_funcs + * @since 4.3.0 */ - def aes_decrypt(input: Column, key: Column): Column = - Column.fn("aes_decrypt", input, key) + def try_variant_array_append(v: Column, path: String, value: Column): Column = + Column.fn("try_variant_array_append", v, lit(path), value) /** - * This is a special version of `aes_decrypt` that performs the same operation, but returns a - * NULL value instead of raising an error if the decryption cannot be performed. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @param aad - * Optional additional authenticated data. Only supported for GCM mode. This can be any - * free-form input and must be provided for both encryption and decryption. A column that - * evaluates to a binary. + * Recursively removes object fields and array elements whose value is a variant null. Returns + * NULL if `v` is NULL. * - * @group misc_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a binary. + * @param v + * a variant column. + * @group variant_funcs + * @since 4.3.0 */ - def try_aes_decrypt( - input: Column, - key: Column, - mode: Column, - padding: Column, - aad: Column): Column = - Column.fn("try_aes_decrypt", input, key, mode, padding, aad) + def variant_strip_nulls(v: Column): Column = Column.fn("variant_strip_nulls", v) /** - * Returns a decrypted value of `input`. + * Recursively removes object fields and array elements whose value is a variant null, unless + * `includeArrays` is false, in which case null array elements are kept. Returns NULL if any + * argument is NULL. * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @param padding - * Specifies how to pad messages whose length is not a multiple of the block size. Valid - * values: PKCS, NONE, DEFAULT. The DEFAULT padding means PKCS for ECB, NONE for GCM and PKCS - * for CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * @param v + * a variant column. + * @param includeArrays + * whether null elements are also removed from arrays. + * @group variant_funcs + * @since 4.3.0 + */ + def variant_strip_nulls(v: Column, includeArrays: Boolean): Column = + Column.fn("variant_strip_nulls", v, lit(includeArrays)) + + /** + * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the extraction path. A valid path should start with `$` and is followed by zero or more + * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the type specified by the `targetType` argument. */ - def try_aes_decrypt(input: Column, key: Column, mode: Column, padding: Column): Column = - Column.fn("try_aes_decrypt", input, key, mode, padding) + def variant_get(v: Column, path: String, targetType: String): Column = + Column.fn("variant_get", v, lit(path), lit(targetType)) /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @param mode - * Specifies which block cipher mode should be used to decrypt messages. Valid modes: ECB, - * GCM, CBC. A column that evaluates to a string. - * @see - * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist. Throws an exception if the cast fails. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the extraction path strings. A valid path string should start with + * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or + * `["name"]`. A column that evaluates to a string. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the type specified by the `targetType` argument. */ - def try_aes_decrypt(input: Column, key: Column, mode: Column): Column = - Column.fn("try_aes_decrypt", input, key, mode) + def variant_get(v: Column, path: Column, targetType: String): Column = + Column.fn("variant_get", v, path, lit(targetType)) /** - * Returns a decrypted value of `input`. - * - * @param input - * The binary value to decrypt. A column that evaluates to a binary. - * @param key - * The passphrase to use to decrypt the data. A column that evaluates to a binary. - * @see - * `org.apache.spark.sql.functions.try_aes_decrypt(Column, Column, Column, Column, Column)` + * Extracts a sub-variant from `v` according to `path` string, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist or the cast fails.. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the extraction path. A valid path should start with `$` and is followed by zero or more + * segments like `[123]`, `.name`, `['name']`, or `["name"]`. A string. Must be a constant. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the type specified by the `targetType` argument. */ - def try_aes_decrypt(input: Column, key: Column): Column = - Column.fn("try_aes_decrypt", input, key) + def try_variant_get(v: Column, path: String, targetType: String): Column = + Column.fn("try_variant_get", v, lit(path), lit(targetType)) /** - * Returns the length of the block being read, or -1 if not available. + * Extracts a sub-variant from `v` according to `path` column, and then cast the sub-variant to + * `targetType`. Returns null if the path does not exist or the cast fails.. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @param path + * the column containing the extraction path strings. A valid path string should start with + * `$` and is followed by zero or more segments like `[123]`, `.name`, `['name']`, or + * `["name"]`. A column that evaluates to a string. + * @param targetType + * the target data type to cast into, in a DDL-formatted string. A string. Must be a constant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column of the type specified by the `targetType` argument. */ - def input_file_block_length(): Column = Column.fn("input_file_block_length") + def try_variant_get(v: Column, path: Column, targetType: String): Column = + Column.fn("try_variant_get", v, lit(path), lit(targetType)) /** - * Returns the start offset of the block being read, or -1 if not available. + * Returns schema in the SQL format of a variant. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def input_file_block_start(): Column = Column.fn("input_file_block_start") + def schema_of_variant(v: Column): Column = Column.fn("schema_of_variant", v) /** - * Calls a method with reflection. + * Returns the merged schema in the SQL format of a variant column. * - * @group misc_funcs - * @since 3.5.0 + * @param v + * a variant column. A column that evaluates to a variant. + * @group variant_funcs + * @since 4.0.0 * @return * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def reflect(cols: Column*): Column = Column.fn("reflect", cols: _*) + def schema_of_variant_agg(v: Column): Column = Column.fn("schema_of_variant_agg", v) /** - * Calls a method with reflection. + * Parses a JSON string and infers its schema in DDL format. + * + * @param json + * a JSON string. A string. Must be a constant. * - * @group misc_funcs - * @since 3.5.0 + * @group json_funcs + * @since 2.4.0 * @return * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def java_method(cols: Column*): Column = Column.fn("java_method", cols: _*) + def schema_of_json(json: String): Column = schema_of_json(lit(json)) /** - * This is a special version of `reflect` that performs the same operation, but returns a NULL - * value instead of raising an error if the invoke method thrown exception. + * Parses a JSON string and infers its schema in DDL format. * - * @group misc_funcs - * @since 4.0.0 + * @param json + * a foldable string column containing a JSON string. A column that evaluates to a string. + * + * @group json_funcs + * @since 2.4.0 * @return * Returns a column that evaluates to a string. */ - @scala.annotation.varargs - def try_reflect(cols: Column*): Column = Column.fn("try_reflect", cols: _*) + def schema_of_json(json: Column): Column = Column.fn("schema_of_json", json) + // scalastyle:off line.size.limit /** - * Returns the Spark version. The string contains 2 fields, the first being a release version - * and the second being a git revision. + * Parses a JSON string and infers its schema in DDL format using options. * - * @group misc_funcs - * @since 3.5.0 + * @param json + * a foldable string column containing JSON data. A column that evaluates to a string. + * @param options + * options to control how the json is parsed. accepts the same options and the json data + * source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * Returns a column that evaluates to a string. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. + * + * @group json_funcs + * @since 3.0.0 */ - def version(): Column = Column.fn("version") + // scalastyle:on line.size.limit + def schema_of_json(json: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("schema_of_json", options.asScala.iterator, json) /** - * Return DDL-formatted type string for the data type of the input. + * Returns the number of elements in the outermost JSON array. `NULL` is returned in case of any + * other valid JSON string, `NULL` or an invalid JSON. * - * @param col - * The value whose data type is returned. A column of any type. - * @group misc_funcs + * @param e + * the JSON array string column. A column that evaluates to a string. + * @group json_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an integer. */ - def typeof(col: Column): Column = Column.fn("typeof", col) + def json_array_length(e: Column): Column = Column.fn("json_array_length", e) /** - * Returns the bit position for the given input column. + * Returns all the keys of the outermost JSON object as an array. If a valid JSON object is + * given, all the keys of the outermost object will be returned as an array. If it is any other + * valid JSON string, an invalid JSON string or an empty string, the function returns null. * - * @param col - * The input column. A column that evaluates to an integral. - * @group misc_funcs + * @param e + * the JSON object string column. A column that evaluates to a string. + * @group json_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an array. */ - def bitmap_bit_position(col: Column): Column = - Column.fn("bitmap_bit_position", col) + def json_object_keys(e: Column): Column = Column.fn("json_object_keys", e) /** - * Returns the bucket number for the given input column. + * Returns the type of the outermost JSON value as a string: one of 'object', 'array', 'string', + * 'number', 'boolean', or 'null'. Returns null for invalid or empty input. * - * @param col - * The input column. A column that evaluates to an integral. - * @group misc_funcs - * @since 3.5.0 + * @param e + * the JSON string column. A column that evaluates to a string. + * @group json_funcs + * @since 4.4.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def bitmap_bucket_number(col: Column): Column = - Column.fn("bitmap_bucket_number", col) + def json_typeof(e: Column): Column = Column.fn("json_typeof", e) + // scalastyle:off line.size.limit /** - * Returns the number of set bits in the input bitmap. + * (Scala-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into + * a JSON string with the specified schema. Throws an exception, in the case of an unsupported + * type. * - * @param col - * The input bitmap. A column that evaluates to a binary. - * @group misc_funcs - * @since 3.5.0 + * @param e + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. + * @param options + * options to control how the struct column is converted into a json string. accepts the same + * options and the json data source. See Data + * Source Option in the version you use. Additionally the function supports the `pretty` + * option which enables pretty JSON generation. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def bitmap_count(col: Column): Column = Column.fn("bitmap_count", col) + // scalastyle:on line.size.limit + def to_json(e: Column, options: Map[String, String]): Column = + Column.fnWithOptions("to_json", options.iterator, e) + // scalastyle:off line.size.limit /** - * Returns a bitmap that is the bitwise AND of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * (Java-specific) Converts a column containing a `StructType`, `ArrayType` or a `MapType` into + * a JSON string with the specified schema. Throws an exception, in the case of an unsupported + * type. * - * @param left - * A column that evaluates to a binary bitmap. - * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 + * @param e + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. + * @param options + * options to control how the struct column is converted into a json string. accepts the same + * options and the json data source. See Data + * Source Option in the version you use. Additionally the function supports the `pretty` + * option which enables pretty JSON generation. A map of string options. Must be a constant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a binary bitmap. + * Returns a column that evaluates to a string. */ - def bitmap_and(left: Column, right: Column): Column = Column.fn("bitmap_and", left, right) + // scalastyle:on line.size.limit + def to_json(e: Column, options: java.util.Map[String, String]): Column = + to_json(e, options.asScala.toMap) /** - * Returns a bitmap that is the bitwise OR of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * Converts a column containing a `StructType`, `ArrayType` or a `MapType` into a JSON string + * with the specified schema. Throws an exception, in the case of an unsupported type. * - * @param left - * A column that evaluates to a binary bitmap. - * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 + * @param e + * a column containing a struct, an array, a map, or a variant. A column that evaluates to a + * struct, array, map, or variant. + * + * @group json_funcs + * @since 2.1.0 * @return - * Returns a column that evaluates to a binary bitmap. + * Returns a column that evaluates to a string. */ - def bitmap_or(left: Column, right: Column): Column = Column.fn("bitmap_or", left, right) + def to_json(e: Column): Column = + to_json(e, Map.empty[String, String]) /** - * Returns a bitmap that is the bitwise AND NOT of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * Masks the given string value. The function replaces characters with 'X' or 'x', and numbers + * with 'n'. This can be useful for creating copies of tables with sensitive information + * removed. * - * @param left - * A column that evaluates to a binary bitmap. - * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary bitmap. + * Returns a column that evaluates to a string. */ - def bitmap_andnot(left: Column, right: Column): Column = - Column.fn("bitmap_andnot", left, right) + def mask(input: Column): Column = Column.fn("mask", input) /** - * Returns a bitmap that is the bitwise XOR of two input bitmaps. The result is always a - * 4096-byte Spark Binary bitmap. If either input is NULL, the result is NULL. Missing bytes in - * shorter inputs are treated as zero, and inputs longer than 4096 bytes raise - * `BITMAP_INPUT_TOO_LARGE`. Both inputs must use the same bit-position mapping. If they were - * constructed by grouping `bitmap_bit_position` values by `bitmap_bucket_number`, they must - * represent the same bucket because the bitmap bytes do not retain bucket metadata. This scalar - * function operates on two bitmaps from the same row; use `bitmap_*_agg` to combine bitmaps - * across rows. The representation is not a RoaringBitmap serialization. + * Masks the given string value. The function replaces upper-case characters with specific + * character, lower-case characters with 'x', and numbers with 'n'. This can be useful for + * creating copies of tables with sensitive information removed. * - * @param left - * A column that evaluates to a binary bitmap. - * @param right - * A column that evaluates to a binary bitmap. - * @group misc_funcs - * @since 4.4.0 + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * + * @group string_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary bitmap. + * Returns a column that evaluates to a string. */ - def bitmap_xor(left: Column, right: Column): Column = Column.fn("bitmap_xor", left, right) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Datasketch Functions - ////////////////////////////////////////////////////////////////////////////////////////////// + def mask(input: Column, upperChar: Column): Column = + Column.fn("mask", input, upperChar) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches HllSketch. + * Masks the given string value. The function replaces upper-case and lower-case characters with + * the characters specified respectively, and numbers with 'n'. This can be useful for creating + * copies of tables with sensitive information removed. + * + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param lowerChar + * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. * - * @param c - * The binary representation of a Datasketches HllSketch. A column that evaluates to a binary. - * @group sketch_funcs + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def hll_sketch_estimate(c: Column): Column = Column.fn("hll_sketch_estimate", c) + def mask(input: Column, upperChar: Column, lowerChar: Column): Column = + Column.fn("mask", input, upperChar, lowerChar) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches HllSketch. + * Masks the given string value. The function replaces upper-case, lower-case characters and + * numbers with the characters specified respectively. This can be useful for creating copies of + * tables with sensitive information removed. * - * @param columnName - * Name of the column containing the binary representation of a Datasketches HllSketch. A - * column that evaluates to a binary. - * @group sketch_funcs - * @since 3.5.0 - * @return - * Returns a column that evaluates to a long. - */ - def hll_sketch_estimate(columnName: String): Column = { - hll_sketch_estimate(Column(columnName)) - } - - /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values. + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param lowerChar + * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param digitChar + * character to replace digit characters with. Specify NULL to retain original character. A + * column that evaluates to a string. * - * @param c1 - * The first binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @group sketch_funcs + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_union(c1: Column, c2: Column): Column = - Column.fn("hll_union", c1, c2) + def mask(input: Column, upperChar: Column, lowerChar: Column, digitChar: Column): Column = + Column.fn("mask", input, upperChar, lowerChar, digitChar) /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values. + * Masks the given string value. This can be useful for creating copies of tables with sensitive + * information removed. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @group sketch_funcs + * @param input + * string value to mask. Supported types: STRING, VARCHAR, CHAR. A column that evaluates to a + * string. + * @param upperChar + * character to replace upper-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param lowerChar + * character to replace lower-case characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * @param digitChar + * character to replace digit characters with. Specify NULL to retain original character. A + * column that evaluates to a string. + * @param otherChar + * character to replace all other characters with. Specify NULL to retain original character. + * A column that evaluates to a string. + * + * @group string_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def hll_union(columnName1: String, columnName2: String): Column = { - hll_union(Column(columnName1), Column(columnName2)) - } + def mask( + input: Column, + upperChar: Column, + lowerChar: Column, + digitChar: Column, + otherChar: Column): Column = + Column.fn("mask", input, upperChar, lowerChar, digitChar, otherChar) /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values and - * allowDifferentLgConfigK is set to false. + * Returns length of array or map. * - * @param c1 - * The first binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches HllSketch. A column that evaluates to a - * binary. - * @param allowDifferentLgConfigK - * Allow sketches with different lgConfigK values to be merged (defaults to false). A column - * that evaluates to a boolean. Must be a constant. - * @group sketch_funcs - * @since 3.5.0 + * This function returns -1 for null input only if spark.sql.ansi.enabled is false and + * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the + * default settings, the function returns null for null input. + * + * @param e + * the target column. A column that evaluates to an array or a map. + * @group collection_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an integer. */ - def hll_union(c1: Column, c2: Column, allowDifferentLgConfigK: Boolean): Column = - Column.fn("hll_union", c1, c2, lit(allowDifferentLgConfigK)) + def size(e: Column): Column = Column.fn("size", e) /** - * Merges two binary representations of Datasketches HllSketch objects, using a Datasketches - * Union object. Throws an exception if sketches have different lgConfigK values and - * allowDifferentLgConfigK is set to false. + * Returns length of array or map. This is an alias of `size` function. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches HllSketch. - * A column that evaluates to a binary. - * @param allowDifferentLgConfigK - * Allow sketches with different lgConfigK values to be merged (defaults to false). A column - * that evaluates to a boolean. Must be a constant. - * @group sketch_funcs + * This function returns -1 for null input only if spark.sql.ansi.enabled is false and + * spark.sql.legacy.sizeOfNull is true. Otherwise, it returns null for null input. With the + * default settings, the function returns null for null input. + * + * @param e + * the target column. A column that evaluates to an array or a map. + * @group collection_funcs * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an integer. */ - def hll_union( - columnName1: String, - columnName2: String, - allowDifferentLgConfigK: Boolean): Column = { - hll_union(Column(columnName1), Column(columnName2), allowDifferentLgConfigK) - } + def cardinality(e: Column): Column = Column.fn("cardinality", e) /** - * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches AnotB object + * Sorts the input array for the given column in ascending order, according to the natural + * ordering of the array elements. Null elements will be placed at the beginning of the returned + * array. * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the array column to sort. A column that evaluates to an array. + * @group array_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def theta_difference(c1: Column, c2: Column): Column = - Column.fn("theta_difference", c1, c2) + def sort_array(e: Column): Column = sort_array(e, asc = true) /** - * Subtracts two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches AnotB object + * Sorts the input array for the given column in ascending or descending order, according to the + * natural ordering of the array elements. NaN is greater than any non-NaN elements for + * double/float type. Null elements will be placed at the beginning of the returned array in + * ascending order or at the end of the returned array in descending order. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the array column to sort. A column that evaluates to an array. + * @param asc + * whether to sort in ascending order. A column that evaluates to a boolean. Must be a + * constant. + * @group array_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def theta_difference(columnName1: String, columnName2: String): Column = { - theta_difference(Column(columnName1), Column(columnName2)) - } + def sort_array(e: Column, asc: Boolean): Column = Column.fn("sort_array", e, lit(asc)) /** - * Intersects two binary representations of Datasketches ThetaSketch objects in the input - * columns using a Datasketches Intersection object + * Returns the minimum value in the array. NaN is greater than any non-NaN elements for + * double/float type. NULL elements are skipped. * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the array column. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the element type of the input array. */ - def theta_intersection(c1: Column, c2: Column): Column = - Column.fn("theta_intersection", c1, c2) + def array_min(e: Column): Column = Column.fn("array_min", e) /** - * Intersects two binary representations of Datasketches ThetaSketch objects in the input - * columns using a Datasketches Intersection object + * Returns the maximum value in the array. NaN is greater than any non-NaN elements for + * double/float type. NULL elements are skipped. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the input column. A column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the element type of the input array. */ - def theta_intersection(columnName1: String, columnName2: String): Column = { - theta_intersection(Column(columnName1), Column(columnName2)) - } + def array_max(e: Column): Column = Column.fn("array_max", e) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches ThetaSketch. + * Returns the total number of elements in the array. The function returns null for null input. * - * @param c - * The binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the input column. A column that evaluates to an array. + * @group array_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an integer. */ - def theta_sketch_estimate(c: Column): Column = Column.fn("theta_sketch_estimate", c) + def array_size(e: Column): Column = Column.fn("array_size", e) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches ThetaSketch. + * Aggregate function: returns a list of objects with duplicates. * - * @param columnName - * Name of the column containing the binary representation of a Datasketches ThetaSketch. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the input column. A column that evaluates to any type. + * @note + * The function is non-deterministic because the order of collected results depends on the + * order of the rows which may be non-deterministic after a shuffle. + * @group agg_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an array. */ - def theta_sketch_estimate(columnName: String): Column = { - theta_sketch_estimate(Column(columnName)) - } + def array_agg(e: Column): Column = Column.fn("array_agg", e) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It is configured with the default value of 12 for - * `lgNomEntries`. + * Returns a random permutation of the given array. * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the input column. A column that evaluates to an array. + * @note + * The function is non-deterministic. + * + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def theta_union(c1: Column, c2: Column): Column = - Column.fn("theta_union", c1, c2) + def shuffle(e: Column): Column = shuffle(e, lit(SparkClassUtils.random.nextLong)) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It is configured with the default value of 12 for - * `lgNomEntries`. + * Returns a random permutation of the given array. * - * @param columnName1 - * Name of the column containing the first binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @param columnName2 - * Name of the column containing the second binary representation of a Datasketches - * ThetaSketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param e + * the input column. A column that evaluates to an array. + * @param seed + * the seed for the random generator. A column that evaluates to an integral. Must be a + * constant. + * @note + * The function is non-deterministic. + * + * @group array_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def theta_union(columnName1: String, columnName2: String): Column = { - theta_union(Column(columnName1), Column(columnName2)) - } + def shuffle(e: Column, seed: Column): Column = Column.fn("shuffle", e, seed) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. - * - * @param c1 - * The first binary representation of a Datasketches ThetaSketch. A column that evaluates to a - * binary. - * @param c2 - * The second binary representation of a Datasketches ThetaSketch. A column that evaluates to - * a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union operation (must be between 4 and 26, - * defaults to 12). A column that evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.1.0 + * Returns a reversed string or an array with reverse order of elements. + * @param e + * the input column. A column that evaluates to a string, a binary, or an array. + * @group collection_funcs + * @since 1.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def theta_union(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("theta_union", c1, c2, lit(lgNomEntries)) + def reverse(e: Column): Column = Column.fn("reverse", e) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. - * - * @param columnName1 - * The first ThetaSketch column to union. A column that evaluates to a binary. - * @param columnName2 - * The second ThetaSketch column to union. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group sketch_funcs - * @since 4.1.0 + * Creates a single array from an array of arrays. If a structure of nested arrays is deeper + * than two levels, only one level of nesting is removed. + * @param e + * the input column. A column that evaluates to an array of arrays. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def theta_union(columnName1: String, columnName2: String, lgNomEntries: Int): Column = { - theta_union(Column(columnName1), Column(columnName2), lgNomEntries) - } + def flatten(e: Column): Column = Column.fn("flatten", e) /** - * Unions two binary representations of Datasketches ThetaSketch objects in the input columns - * using a Datasketches Union object. It allows the configuration of `lgNomEntries` log nominal - * entries for the union buffer. + * Generate a sequence of integers from start to stop, incrementing by step. * - * @param c1 - * The first ThetaSketch column to union. A column that evaluates to a binary. - * @param c2 - * The second ThetaSketch column to union. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. - * @group sketch_funcs - * @since 4.1.0 + * @param start + * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a + * date, or a timestamp. + * @param stop + * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, + * or a timestamp. + * @param step + * the value to add to the current element to get the next element. A column that evaluates to + * an integral or interval. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def theta_union(c1: Column, c2: Column, lgNomEntries: Column): Column = - Column.fn("theta_union", c1, c2, lgNomEntries) + def sequence(start: Column, stop: Column, step: Column): Column = + Column.fn("sequence", start, stop, step) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. + * Generate a sequence of integers from start to stop, incrementing by 1 if start is less than + * or equal to stop, otherwise -1. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param start + * the starting value (inclusive) of the sequence. A column that evaluates to an integral, a + * date, or a timestamp. + * @param stop + * the last value (inclusive) of the sequence. A column that evaluates to an integral, a date, + * or a timestamp. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def tuple_difference_double(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_double", c1, c2) + def sequence(start: Column, stop: Column): Column = Column.fn("sequence", start, stop) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. + * Creates an array containing the left argument repeated the number of times given by the right + * argument. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param left + * the value to repeat. A column that evaluates to any type. + * @param right + * the number of times to repeat the value. A column that evaluates to an integral. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def tuple_difference_double(columnName1: String, columnName2: String): Column = - tuple_difference_double(Column(columnName1), Column(columnName2)) + def array_repeat(left: Column, right: Column): Column = Column.fn("array_repeat", left, right) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. + * Creates an array containing the left argument repeated the number of times given by the right + * argument. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * the value to repeat. A column that evaluates to any type. + * @param count + * the number of times to repeat the value. A column that evaluates to an integral. Must be a + * constant. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def tuple_difference_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_integer", c1, c2) + def array_repeat(e: Column, count: Int): Column = array_repeat(e, lit(count)) /** - * Subtracts two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches AnotB object. Returns elements in the - * first sketch that are not in the second sketch. - * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to subtract. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * Returns true if the map contains the key. + * @param column + * the input column. A column that evaluates to a map. + * @param key + * the key to check for. A column that evaluates to the map's key type. Must be a constant. + * @group map_funcs + * @since 3.3.0 + * @return + * Returns a column that evaluates to a boolean. + */ + def map_contains_key(column: Column, key: Any): Column = + Column.fn("map_contains_key", column, lit(key)) + + /** + * Returns an unordered array containing the keys of the map. + * @param e + * the input column. A column that evaluates to a map. + * @group map_funcs + * @since 2.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def map_keys(e: Column): Column = Column.fn("map_keys", e) + + /** + * Returns an unordered array containing the values of the map. + * @param e + * the input column. A column that evaluates to a map. + * @group map_funcs + * @since 2.3.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def tuple_difference_integer(columnName1: String, columnName2: String): Column = - tuple_difference_integer(Column(columnName1), Column(columnName2)) + def map_values(e: Column): Column = Column.fn("map_values", e) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). It is configured with the default mode of 'sum'. - * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * Returns an unordered array of all entries in the given map. + * @param e + * the input column. A column that evaluates to a map. + * @group map_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def tuple_intersection_double(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_double", c1, c2) + def map_entries(e: Column): Column = Column.fn("map_entries", e) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). It is configured with the default mode of 'sum'. - * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * Returns a map created from the given array of entries. + * @param e + * the array of entries to convert. A column that evaluates to an array of structs, each with + * a key and value field. + * @group map_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a map. */ - def tuple_intersection_double(columnName1: String, columnName2: String): Column = - tuple_intersection_double(Column(columnName1), Column(columnName2)) + def map_from_entries(e: Column): Column = Column.fn("map_from_entries", e) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). - * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * Returns a merged array of structs in which the N-th struct contains all N-th values of input + * arrays. + * @param e + * the columns of arrays to be merged. Each is a column that evaluates to an array. + * @group array_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an array. */ - def tuple_intersection_double(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_double", c1, c2, lit(mode)) + @scala.annotation.varargs + def arrays_zip(e: Column*): Column = Column.fn("arrays_zip", e: _*) /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). - * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * Returns the union of all the given maps. + * @param cols + * the maps to merge. Each is a column that evaluates to a map. + * @group map_funcs + * @since 2.4.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a map. */ - def tuple_intersection_double(columnName1: String, columnName2: String, mode: String): Column = - tuple_intersection_double(Column(columnName1), Column(columnName2), mode) + @scala.annotation.varargs + def map_concat(cols: Column*): Column = Column.fn("map_concat", cols: _*) + // scalastyle:off line.size.limit /** - * Intersects two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Intersection object. The mode parameter - * specifies the aggregation mode for numeric summaries during intersection (sum, min, max, - * alwaysone). + * Parses a column containing a CSV string into a `StructType` with the specified schema. + * Returns `null`, in the case of an unparseable string. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a string column containing CSV data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the CSV string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the CSV is parsed. accepts the same options and the CSV data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a struct. */ - def tuple_intersection_double(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_double", c1, c2, mode) + // scalastyle:on line.size.limit + def from_csv(e: Column, schema: StructType, options: Map[String, String]): Column = + from_csv(e, lit(schema.toDDL), options.iterator) + // scalastyle:off line.size.limit /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). It is configured with the default mode of 'sum'. + * (Java-specific) Parses a column containing a CSV string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a string column containing CSV data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the CSV string. A column that evaluates to a string. + * @param options + * options to control how the CSV is parsed. accepts the same options and the CSV data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a struct. */ - def tuple_intersection_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_integer", c1, c2) + // scalastyle:on line.size.limit + def from_csv(e: Column, schema: Column, options: java.util.Map[String, String]): Column = + from_csv(e, schema, options.asScala.iterator) + + private def from_csv(e: Column, schema: Column, options: Iterator[(String, String)]): Column = + Column.fnWithOptions("from_csv", options, e, schema) /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). It is configured with the default mode of 'sum'. + * Parses a CSV string and infers its schema in DDL format. * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param csv + * a CSV string. A string. Must be a constant. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def tuple_intersection_integer(columnName1: String, columnName2: String): Column = - tuple_intersection_integer(Column(columnName1), Column(columnName2)) + def schema_of_csv(csv: String): Column = schema_of_csv(lit(csv)) /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). + * Parses a CSV string and infers its schema in DDL format. * - * @param c1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param csv + * a foldable string column containing a CSV string. A column that evaluates to a string. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def tuple_intersection_integer(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_integer", c1, c2, lit(mode)) + def schema_of_csv(csv: Column): Column = schema_of_csv(csv, Collections.emptyMap()) + // scalastyle:off line.size.limit /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). + * Parses a CSV string and infers its schema in DDL format using options. * - * @param columnName1 - * The first TupleSketch column to intersect. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column to intersect. A column that evaluates to a binary. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param csv + * a foldable string column containing a CSV string. A column that evaluates to a string. + * @param options + * options to control how the CSV is parsed. accepts the same options and the CSV data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * Returns a column that evaluates to a binary. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. + * @group csv_funcs + * @since 3.0.0 */ - def tuple_intersection_integer(columnName1: String, columnName2: String, mode: String): Column = - tuple_intersection_integer(Column(columnName1), Column(columnName2), mode) + // scalastyle:on line.size.limit + def schema_of_csv(csv: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("schema_of_csv", options.asScala.iterator, csv) + // scalastyle:off line.size.limit /** - * Intersects two binary representations of Datasketches TupleSketch objects with integer - * summary data type in the input columns using a Datasketches Intersection object. The mode - * parameter specifies the aggregation mode for numeric summaries during intersection (sum, min, - * max, alwaysone). + * (Java-specific) Converts a column containing a `StructType` into a CSV string with the + * specified schema. Throws an exception, in the case of an unsupported type. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a column containing a struct. A column that evaluates to a string. + * @param options + * options to control how the struct column is converted into a CSV string. It accepts the + * same options and the CSV data source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def tuple_intersection_integer(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_integer", c1, c2, mode) + // scalastyle:on line.size.limit + def to_csv(e: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("to_csv", options.asScala.iterator, e) /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with double summary data type. + * Converts a column containing a `StructType` into a CSV string with the specified schema. + * Throws an exception, in the case of an unsupported type. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a column containing a struct. A column that evaluates to a string. + * + * @group csv_funcs + * @since 3.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def tuple_sketch_estimate_double(c: Column): Column = - Column.fn("tuple_sketch_estimate_double", c) + def to_csv(e: Column): Column = to_csv(e, Map.empty[String, String].asJava) + // scalastyle:off line.size.limit /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with double summary data type. + * Parses a column containing a XML string into the data type corresponding to the specified + * schema. Returns `null`, in the case of an unparseable string. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a + * constant. + * @param options + * options to control how the XML is parsed. accepts the same options and the XML data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a struct. */ - def tuple_sketch_estimate_double(columnName: String): Column = - tuple_sketch_estimate_double(Column(columnName)) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: StructType, options: java.util.Map[String, String]): Column = + from_xml(e, lit(schema.sql), options.asScala.iterator) + // scalastyle:off line.size.limit /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with integer summary data type. + * (Java-specific) Parses a column containing a XML string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema as a DDL-formatted string. A string, StructType or DataType. Must be a constant. + * @param options + * options to control how the XML is parsed. accepts the same options and the xml data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a struct. */ - def tuple_sketch_estimate_integer(c: Column): Column = - Column.fn("tuple_sketch_estimate_integer", c) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: String, options: java.util.Map[String, String]): Column = { + from_xml(e, lit(schema), options) + } + // scalastyle:off line.size.limit /** - * Returns the estimated number of unique values given the binary representation of a - * Datasketches TupleSketch with integer summary data type. + * (Java-specific) Parses a column containing a XML string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A column that evaluates to a string. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a struct. */ - def tuple_sketch_estimate_integer(columnName: String): Column = - tuple_sketch_estimate_integer(Column(columnName)) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: Column): Column = { + from_xml(e, schema, Iterator.empty) + } + // scalastyle:off line.size.limit /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * (Java-specific) Parses a column containing a XML string into a `StructType` with the + * specified schema. Returns `null`, in the case of an unparseable string. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A column that evaluates to a string. + * @param options + * options to control how the XML is parsed. accepts the same options and the XML data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a struct. */ - def tuple_sketch_summary_double(c: Column): Column = - Column.fn("tuple_sketch_summary_double", c) + // scalastyle:on line.size.limit + def from_xml(e: Column, schema: Column, options: java.util.Map[String, String]): Column = + from_xml(e, schema, options.asScala.iterator) /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * Parses a column containing a XML string into the data type corresponding to the specified + * schema. Returns `null`, in the case of an unparseable string. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a string column containing XML data. A column that evaluates to a string. + * @param schema + * the schema to use when parsing the XML string. A string, StructType or DataType. Must be a + * constant. + * + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a struct. */ - def tuple_sketch_summary_double(columnName: String): Column = - tuple_sketch_summary_double(Column(columnName)) + def from_xml(e: Column, schema: StructType): Column = + from_xml(e, schema, Map.empty[String, String].asJava) + + private def from_xml(e: Column, schema: Column, options: Iterator[(String, String)]): Column = { + Column.fnWithOptions("from_xml", options, e, schema) + } /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Parses a XML string and infers its schema in DDL format. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * a XML string. A string. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def tuple_sketch_summary_double(c: Column, mode: String): Column = - Column.fn("tuple_sketch_summary_double", c, lit(mode)) + def schema_of_xml(xml: String): Column = schema_of_xml(lit(xml)) /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Parses a XML string and infers its schema in DDL format. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * a foldable string column containing a XML string. A column that evaluates to a string. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a string. */ - def tuple_sketch_summary_double(columnName: String, mode: String): Column = - tuple_sketch_summary_double(Column(columnName), mode) + def schema_of_xml(xml: Column): Column = Column.fn("schema_of_xml", xml) + + // scalastyle:off line.size.limit /** - * Aggregates the summary values from a Datasketches TupleSketch with double summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * Parses a XML string and infers its schema in DDL format using options. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * a foldable string column containing XML data. A column that evaluates to a string. + * @param options + * options to control how the xml is parsed. accepts the same options and the XML data source. + * See Data + * Source Option in the version you use. A map of string options. Must be a constant. * @return - * Returns a column that evaluates to a double. + * a column with string literal containing schema in DDL format. Returns a column that + * evaluates to a string. + * @group xml_funcs + * @since 4.0.0 */ - def tuple_sketch_summary_double(c: Column, mode: Column): Column = - Column.fn("tuple_sketch_summary_double", c, mode) + // scalastyle:on line.size.limit + def schema_of_xml(xml: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("schema_of_xml", options.asScala.iterator, xml) + + // scalastyle:off line.size.limit /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * (Java-specific) Converts a column containing a `StructType` into a XML string with the + * specified schema. Throws an exception, in the case of an unsupported type. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a column containing a struct. A column that evaluates to a string. + * @param options + * options to control how the struct column is converted into a XML string. It accepts the + * same options as the XML data source. See Data + * Source Option in the version you use. A map of string options. Must be a constant. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def tuple_sketch_summary_integer(c: Column): Column = - Column.fn("tuple_sketch_summary_integer", c) + // scalastyle:on line.size.limit + def to_xml(e: Column, options: java.util.Map[String, String]): Column = + Column.fnWithOptions("to_xml", options.asScala.iterator, e) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). It is - * configured with the default mode of 'sum'. + * Converts a column containing a `StructType` into a XML string with the specified schema. + * Throws an exception, in the case of an unsupported type. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param e + * a column containing a struct. A column that evaluates to a string. + * @group xml_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to a string. */ - def tuple_sketch_summary_integer(columnName: String): Column = - tuple_sketch_summary_integer(Column(columnName)) + def to_xml(e: Column): Column = to_xml(e, Map.empty[String, String].asJava) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * (Java-specific) A transform for timestamps and dates to partition data into years. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a long. + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def tuple_sketch_summary_integer(c: Column, mode: String): Column = - Column.fn("tuple_sketch_summary_integer", c, lit(mode)) + def years(e: Column): Column = partitioning.years(e) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * (Java-specific) A transform for timestamps and dates to partition data into months. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a long. + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def tuple_sketch_summary_integer(columnName: String, mode: String): Column = - tuple_sketch_summary_integer(Column(columnName), mode) + def months(e: Column): Column = partitioning.months(e) /** - * Aggregates the summary values from a Datasketches TupleSketch with integer summary data type. - * The mode parameter specifies the aggregation mode (sum, min, max, alwaysone). + * (Java-specific) A transform for timestamps and dates to partition data into days. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a long. + * @param e + * the target column to transform. A column that evaluates to a date or a timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def tuple_sketch_summary_integer(c: Column, mode: Column): Column = - Column.fn("tuple_sketch_summary_integer", c, mode) + def days(e: Column): Column = partitioning.days(e) /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. + * Returns a string array of values within the nodes of xml that match the XPath expression. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to an array. */ - def tuple_sketch_theta_double(c: Column): Column = - Column.fn("tuple_sketch_theta_double", c) + def xpath(xml: Column, path: Column): Column = + Column.fn("xpath", xml, path) /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with double summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. + * Returns true if the XPath expression evaluates to true, or if a matching node is found. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double. + * Returns a column that evaluates to a boolean. */ - def tuple_sketch_theta_double(columnName: String): Column = - tuple_sketch_theta_double(Column(columnName)) + def xpath_boolean(xml: Column, path: Column): Column = + Column.fn("xpath_boolean", xml, path) /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. + * Returns a double value, the value zero if no match is found, or NaN if a match is found but + * the value is non-numeric. * - * @param c - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to a double. */ - def tuple_sketch_theta_integer(c: Column): Column = - Column.fn("tuple_sketch_theta_integer", c) + def xpath_double(xml: Column, path: Column): Column = + Column.fn("xpath_double", xml, path) /** - * Returns the theta value (sampling rate) from a Datasketches TupleSketch with integer summary - * data type. The theta value represents the effective sampling rate of the sketch, between 0.0 - * and 1.0. + * Returns a double value, the value zero if no match is found, or NaN if a match is found but + * the value is non-numeric. * - * @param columnName - * The column containing a binary TupleSketch representation. A column that evaluates to a - * binary. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return * Returns a column that evaluates to a double. */ - def tuple_sketch_theta_integer(columnName: String): Column = - tuple_sketch_theta_integer(Column(columnName)) + def xpath_number(xml: Column, path: Column): Column = + Column.fn("xpath_number", xml, path) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Returns a float value, the value zero if no match is found, or NaN if a match is found but + * the value is non-numeric. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a float. */ - def tuple_union_double(c1: Column, c2: Column): Column = - Column.fn("tuple_union_double", c1, c2) + def xpath_float(xml: Column, path: Column): Column = + Column.fn("xpath_float", xml, path) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Returns an integer value, or the value zero if no match is found, or a match is found but the + * value is non-numeric. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an integer. */ - def tuple_union_double(columnName1: String, columnName2: String): Column = - tuple_union_double(Column(columnName1), Column(columnName2)) + def xpath_int(xml: Column, path: Column): Column = + Column.fn("xpath_int", xml, path) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. - * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * Returns a long integer value, or the value zero if no match is found, or a match is found but + * the value is non-numeric. + * + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a long. */ - def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries)) + def xpath_long(xml: Column, path: Column): Column = + Column.fn("xpath_long", xml, path) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. + * Returns a short integer value, or the value zero if no match is found, or a match is found + * but the value is non-numeric. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a short. */ - def tuple_union_double(columnName1: String, columnName2: String, lgNomEntries: Int): Column = - tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries) + def xpath_short(xml: Column, path: Column): Column = + Column.fn("xpath_short", xml, path) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Returns the text contents of the first xml node that matches the XPath expression. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param xml + * the XML column to evaluate. A column that evaluates to a string. + * @param path + * the XPath expression to match. A column that evaluates to a string. Must be a constant. + * @group xml_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a string. */ - def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_double", c1, c2, lit(lgNomEntries), lit(mode)) + def xpath_string(xml: Column, path: Column): Column = + Column.fn("xpath_string", xml, path) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * (Java-specific) A transform for timestamps to partition data into hours. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a binary. + * @param e + * target date or timestamp column to work on. A column that evaluates to a date or timestamp. + * @group partition_transforms + * @since 3.0.0 */ - def tuple_union_double( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) + def hours(e: Column): Column = partitioning.hours(e) /** - * Unions two binary representations of Datasketches TupleSketch objects with double summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Converts the timestamp without time zone `sourceTs` from the `sourceTz` time zone to + * `targetTz`. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. - * @param mode - * The aggregation mode for numeric summaries (sum, min, max, alwaysone). A column that - * evaluates to a string. - * @group sketch_funcs - * @since 4.2.0 + * @param sourceTz + * the time zone for the input timestamp. If it is missed, the current session time zone is + * used as the source time zone. A column that evaluates to a string. + * @param targetTz + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. + * @param sourceTs + * a timestamp without time zone. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_double(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_double", c1, c2, lgNomEntries, mode) + def convert_timezone(sourceTz: Column, targetTz: Column, sourceTs: Column): Column = + Column.fn("convert_timezone", sourceTz, targetTz, sourceTs) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Converts the timestamp without time zone `sourceTs` from the current time zone to `targetTz`. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param targetTz + * the time zone to which the input timestamp should be converted. A column that evaluates to + * a string. + * @param sourceTs + * a timestamp without time zone. A column that evaluates to a timestamp. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_union_integer", c1, c2) + def convert_timezone(targetTz: Column, sourceTs: Column): Column = + Column.fn("convert_timezone", targetTz, sourceTs) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It is configured with the - * default values of 12 for `lgNomEntries` and 'sum' for mode. + * Make DayTimeIntervalType duration from days, hours, mins and secs. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_union_integer(columnName1: String, columnName2: String): Column = - tuple_union_integer(Column(columnName1), Column(columnName2)) + def make_dt_interval(days: Column, hours: Column, mins: Column, secs: Column): Column = + Column.fn("make_dt_interval", days, hours, mins, secs) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. + * Make DayTimeIntervalType duration from days, hours and mins. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries)) + def make_dt_interval(days: Column, hours: Column, mins: Column): Column = + Column.fn("make_dt_interval", days, hours, mins) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer. It uses the default mode of - * 'sum'. + * Make DayTimeIntervalType duration from days and hours. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries for the union buffer. A column that evaluates to an - * integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_union_integer(columnName1: String, columnName2: String, lgNomEntries: Int): Column = - tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries) + def make_dt_interval(days: Column, hours: Column): Column = + Column.fn("make_dt_interval", days, hours) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). - * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a - * constant. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * Make DayTimeIntervalType duration from days. + * + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_integer", c1, c2, lit(lgNomEntries), lit(mode)) + def make_dt_interval(days: Column): Column = + Column.fn("make_dt_interval", days) /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * Make DayTimeIntervalType duration. * - * @param columnName1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries. A column that evaluates to an integral. Must be a - * constant. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_union_integer( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) + def make_dt_interval(): Column = + Column.fn("make_dt_interval") /** - * Unions two binary representations of Datasketches TupleSketch objects with integer summary - * data type in the input columns using a Datasketches Union object. It allows the configuration - * of `lgNomEntries` log nominal entries for the union buffer and the aggregation mode for - * numeric summaries (sum, min, max, alwaysone). + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param c1 - * The first TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The second TupleSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries. A column that evaluates to an integral. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_union_integer(c1: Column, c2: Column, lgNomEntries: Column, mode: Column): Column = - Column.fn("tuple_union_integer", c1, c2, lgNomEntries, mode) + def try_make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days, hours, mins, secs) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * double summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * Make interval from years, months, weeks, days, hours, mins and secs. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @param secs + * the number of seconds with the fractional part in microsecond precision. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_difference_theta_double(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_theta_double", c1, c2) + def make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_interval", years, months, weeks, days, hours, mins, secs) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * double summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_difference_theta_double(columnName1: String, columnName2: String): Column = - tuple_difference_theta_double(Column(columnName1), Column(columnName2)) + def try_make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days, hours, mins) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * integer summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * Make interval from years, months, weeks, days, hours and mins. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @param mins + * the number of minutes, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_difference_theta_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_difference_theta_integer", c1, c2) + def make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column, + mins: Column): Column = + Column.fn("make_interval", years, months, weeks, days, hours, mins) /** - * Subtracts the binary representation of a Datasketches ThetaSketch from a TupleSketch with - * integer summary data type in the input columns using a Datasketches AnotB object. Returns - * elements in the TupleSketch that are not in the ThetaSketch. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_difference_theta_integer(columnName1: String, columnName2: String): Column = - tuple_difference_theta_integer(Column(columnName1), Column(columnName2)) + def try_make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days, hours) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * Make interval from years, months, weeks, days and hours. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @param hours + * the number of hours, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_double(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_theta_double", c1, c2) + def make_interval( + years: Column, + months: Column, + weeks: Column, + days: Column, + hours: Column): Column = + Column.fn("make_interval", years, months, weeks, days, hours) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_double(columnName1: String, columnName2: String): Column = - tuple_intersection_theta_double(Column(columnName1), Column(columnName2)) + def try_make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = + Column.fn("try_make_interval", years, months, weeks, days) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Make interval from years, months, weeks and days. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @param days + * the number of days, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_double(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_theta_double", c1, c2, lit(mode)) + def make_interval(years: Column, months: Column, weeks: Column, days: Column): Column = + Column.fn("make_interval", years, months, weeks, days) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * the number of years, positive or negative. A column that evaluates to an integral. + * @param months + * the number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * the number of weeks, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_double( - columnName1: String, - columnName2: String, - mode: String): Column = - tuple_intersection_theta_double(Column(columnName1), Column(columnName2), mode) + def try_make_interval(years: Column, months: Column, weeks: Column): Column = + Column.fn("try_make_interval", years, months, weeks) /** - * Intersects the binary representation of a Datasketches TupleSketch with double summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Make interval from years, months and weeks. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @param weeks + * The number of weeks, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_double(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_theta_double", c1, c2, mode) + def make_interval(years: Column, months: Column, weeks: Column): Column = + Column.fn("make_interval", years, months, weeks) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_intersection_theta_integer", c1, c2) + def try_make_interval(years: Column, months: Column): Column = + Column.fn("try_make_interval", years, months) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). It is configured with the default mode of 'sum'. + * Make interval from years and months. * - * @param columnName1 - * The TupleSketch column. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_integer(columnName1: String, columnName2: String): Column = - tuple_intersection_theta_integer(Column(columnName1), Column(columnName2)) + def make_interval(years: Column, months: Column): Column = + Column.fn("make_interval", years, months) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * This is a special version of `make_interval` that performs the same operation, but returns a + * NULL value instead of raising an error if interval cannot be created. * - * @param c1 - * The TupleSketch column. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum", "min", "max", or "alwaysone". A column that evaluates to a string. - * Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: String): Column = - Column.fn("tuple_intersection_theta_integer", c1, c2, lit(mode)) + def try_make_interval(years: Column): Column = + Column.fn("try_make_interval", years) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Make interval from years. * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_integer( - columnName1: String, - columnName2: String, - mode: String): Column = - tuple_intersection_theta_integer(Column(columnName1), Column(columnName2), mode) + def make_interval(years: Column): Column = + Column.fn("make_interval", years) /** - * Intersects the binary representation of a Datasketches TupleSketch with integer summary data - * type with a Datasketches ThetaSketch in the input columns using a Datasketches Intersection - * object. The mode parameter specifies the aggregation mode for numeric summaries during - * intersection (sum, min, max, alwaysone). + * Make interval. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to an interval. */ - def tuple_intersection_theta_integer(c1: Column, c2: Column, mode: Column): Column = - Column.fn("tuple_intersection_theta_integer", c1, c2, mode) + def make_interval(): Column = + Column.fn("make_interval") /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Create timestamp from years, months, days, hours, mins, secs and timezone fields. The result + * data type is consistent with the value of configuration `spark.sql.timestampType`. If the + * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. + * Otherwise, it will throw an error instead. * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_double(c1: Column, c2: Column): Column = - Column.fn("tuple_union_theta_double", c1, c2) + def make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("make_timestamp", years, months, days, hours, mins, secs, timezone) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Create timestamp from years, months, days, hours, mins and secs fields. The result data type + * is consistent with the value of configuration `spark.sql.timestampType`. If the configuration + * `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. Otherwise, it + * will throw an error instead. * - * @param columnName1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_double(columnName1: String, columnName2: String): Column = - tuple_union_theta_double(Column(columnName1), Column(columnName2)) + def make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_timestamp", years, months, days, hours, mins, secs) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Create a local date-time from date, time, and timezone fields. * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries)) + def make_timestamp(date: Column, time: Column, timezone: Column): Column = + Column.fn("make_timestamp", date, time, timezone) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Create a local date-time from date and time fields. * - * @param columnName1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_double( - columnName1: String, - columnName2: String, - lgNomEntries: Int): Column = - tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries) + def make_timestamp(date: Column, time: Column): Column = + Column.fn("make_timestamp", date, time) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Try to create a timestamp from years, months, days, hours, mins, secs and timezone fields. + * The result data type is consistent with the value of configuration `spark.sql.timestampType`. + * The function returns NULL on invalid inputs. * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_double(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_theta_double", c1, c2, lit(lgNomEntries), lit(mode)) + def try_make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("try_make_timestamp", years, months, days, hours, mins, secs, timezone) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Try to create a timestamp from years, months, days, hours, mins, and secs fields. The result + * data type is consistent with the value of configuration `spark.sql.timestampType`. The + * function returns NULL on invalid inputs. * - * @param columnName1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_double( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_theta_double(Column(columnName1), Column(columnName2), lgNomEntries, mode) + def try_make_timestamp( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_timestamp", years, months, days, hours, mins, secs) /** - * Unions the binary representation of a Datasketches TupleSketch with double summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Try to create a local date-time from date, time, and timezone fields. * - * @param c1 - * The TupleSketch column with double summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. - * @group sketch_funcs - * @since 4.2.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_double( - c1: Column, - c2: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_union_theta_double", c1, c2, lgNomEntries, mode) + def try_make_timestamp(date: Column, time: Column, timezone: Column): Column = + Column.fn("try_make_timestamp", date, time, timezone) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Try to create a local date-time from date and time fields. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_integer(c1: Column, c2: Column): Column = - Column.fn("tuple_union_theta_integer", c1, c2) + def try_make_timestamp(date: Column, time: Column): Column = + Column.fn("try_make_timestamp", date, time) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It is - * configured with the default values of 12 for `lgNomEntries` and 'sum' for mode. + * Create the current timestamp with local time zone from years, months, days, hours, mins, secs + * and timezone fields. If the configuration `spark.sql.ansi.enabled` is false, the function + * returns NULL on invalid inputs. Otherwise, it will throw an error instead. * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 to 12. A column that evaluates to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_integer(columnName1: String, columnName2: String): Column = - tuple_union_theta_integer(Column(columnName1), Column(columnName2)) + def make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Create the current timestamp with local time zone from years, months, days, hours, mins and + * secs fields. If the configuration `spark.sql.ansi.enabled` is false, the function returns + * NULL on invalid inputs. Otherwise, it will throw an error instead. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int): Column = - Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries)) + def make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_timestamp_ltz", years, months, days, hours, mins, secs) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer. It uses - * the default mode of 'sum'. + * Try to create the current timestamp with local time zone from years, months, days, hours, + * mins, secs and timezone fields. The function returns NULL on invalid inputs. * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @param timezone + * The time zone identifier. A column that evaluates to a string. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_integer( - columnName1: String, - columnName2: String, - lgNomEntries: Int): Column = - tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries) + def try_make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column, + timezone: Column): Column = + Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs, timezone) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Try to create the current timestamp with local time zone from years, months, days, hours, + * mins and secs fields. The function returns NULL on invalid inputs. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_integer(c1: Column, c2: Column, lgNomEntries: Int, mode: String): Column = - Column.fn("tuple_union_theta_integer", c1, c2, lit(lgNomEntries), lit(mode)) + def try_make_timestamp_ltz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_timestamp_ltz", years, months, days, hours, mins, secs) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Create local date-time from years, months, days, hours, mins, secs fields. If the + * configuration `spark.sql.ansi.enabled` is false, the function returns NULL on invalid inputs. + * Otherwise, it will throw an error instead. * - * @param columnName1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param columnName2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: sum (default), min, max, or alwaysone. A column that evaluates to a - * string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 - * @return - * Returns a column that evaluates to a binary. - */ - def tuple_union_theta_integer( - columnName1: String, - columnName2: String, - lgNomEntries: Int, - mode: String): Column = - tuple_union_theta_integer(Column(columnName1), Column(columnName2), lgNomEntries, mode) + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 3.5.0 + * @return + * Returns a column that evaluates to a timestamp. + */ + def make_timestamp_ntz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("make_timestamp_ntz", years, months, days, hours, mins, secs) /** - * Unions the binary representation of a Datasketches TupleSketch with integer summary data type - * with a Datasketches ThetaSketch in the input columns using a Datasketches Union object. It - * allows the configuration of `lgNomEntries` log nominal entries for the union buffer and the - * aggregation mode for numeric summaries (sum, min, max, alwaysone). + * Create a local date-time from date and time fields. * - * @param c1 - * The TupleSketch column with integer summaries. A column that evaluates to a binary. - * @param c2 - * The ThetaSketch column. A column that evaluates to a binary. - * @param lgNomEntries - * The log-base-2 of nominal entries (must be between 4 and 26, defaults to 12). A column that - * evaluates to an integral. Must be a constant. - * @param mode - * The summary mode: "sum" (default), "min", "max", or "alwaysone". A column that evaluates to - * a string. Must be a constant. - * @group sketch_funcs - * @since 4.2.0 + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a timestamp. */ - def tuple_union_theta_integer( - c1: Column, - c2: Column, - lgNomEntries: Column, - mode: Column): Column = - Column.fn("tuple_union_theta_integer", c1, c2, lgNomEntries, mode) + def make_timestamp_ntz(date: Column, time: Column): Column = + Column.fn("make_timestamp_ntz", date, time) /** - * Returns a string with human readable summary information about the KLL bigint sketch. + * Try to create a local date-time from years, months, days, hours, mins, secs fields. The + * function returns NULL on invalid inputs. * - * @param e - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param years + * The year to represent, from 1 to 9999. A column that evaluates to an integral. + * @param months + * The month-of-year to represent, from 1 (January) to 12 (December). A column that evaluates + * to an integral. + * @param days + * The day-of-month to represent, from 1 to 31. A column that evaluates to an integral. + * @param hours + * The hour-of-day to represent, from 0 to 23. A column that evaluates to an integral. + * @param mins + * The minute-of-hour to represent, from 0 to 59. A column that evaluates to an integral. + * @param secs + * The second-of-minute and its micro-fraction to represent, from 0 to 60. A column that + * evaluates to a numeric. + * @group datetime_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def kll_sketch_to_string_bigint(e: Column): Column = - Column.fn("kll_sketch_to_string_bigint", e) + def try_make_timestamp_ntz( + years: Column, + months: Column, + days: Column, + hours: Column, + mins: Column, + secs: Column): Column = + Column.fn("try_make_timestamp_ntz", years, months, days, hours, mins, secs) /** - * Returns a string with human readable summary information about the KLL float sketch. + * Try to create a local date-time from date and time fields. * - * @param e - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs + * @param date + * The date to represent, in valid DATE format. A column that evaluates to a date. + * @param time + * The time to represent, in valid TIME format. A column that evaluates to a time. + * @group datetime_funcs * @since 4.1.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to a timestamp. */ - def kll_sketch_to_string_float(e: Column): Column = - Column.fn("kll_sketch_to_string_float", e) + def try_make_timestamp_ntz(date: Column, time: Column): Column = + Column.fn("try_make_timestamp_ntz", date, time) /** - * Returns a string with human readable summary information about the KLL double sketch. + * Make year-month interval from years, months. * - * @param e - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @param months + * The number of months, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a string. + * Returns a column that evaluates to an interval. */ - def kll_sketch_to_string_double(e: Column): Column = - Column.fn("kll_sketch_to_string_double", e) + def make_ym_interval(years: Column, months: Column): Column = + Column.fn("make_ym_interval", years, months) /** - * Returns the number of items collected in the KLL bigint sketch. + * Make year-month interval from years. * - * @param e - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param years + * The number of years, positive or negative. A column that evaluates to an integral. + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an interval. */ - def kll_sketch_get_n_bigint(e: Column): Column = - Column.fn("kll_sketch_get_n_bigint", e) + def make_ym_interval(years: Column): Column = Column.fn("make_ym_interval", years) /** - * Returns the number of items collected in the KLL float sketch. + * Make year-month interval. * - * @param e - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @group datetime_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long. + * Returns a column that evaluates to an interval. */ - def kll_sketch_get_n_float(e: Column): Column = - Column.fn("kll_sketch_get_n_float", e) + def make_ym_interval(): Column = Column.fn("make_ym_interval") /** - * Returns the number of items collected in the KLL double sketch. + * (Java-specific) A transform for any type that partitions by a hash of the input column. * + * @param numBuckets + * The number of buckets. A column that evaluates to an integral. Must be a constant. * @param e - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 - * @return - * Returns a column that evaluates to a long. + * The input column to partition. A column of any type. + * @group partition_transforms + * @since 3.0.0 */ - def kll_sketch_get_n_double(e: Column): Column = - Column.fn("kll_sketch_get_n_double", e) + def bucket(numBuckets: Column, e: Column): Column = partitioning.bucket(numBuckets, e) /** - * Merges two KLL bigint sketch buffers together into one. + * (Java-specific) A transform for any type that partitions by a hash of the input column. * - * @param left - * The first KLL bigint sketch. A column that evaluates to a binary. - * @param right - * The second KLL bigint sketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 - * @return - * Returns a column that evaluates to a binary. + * @param numBuckets + * The number of buckets. Must be a constant. + * @param e + * The input column to partition. A column of any type. + * @group partition_transforms + * @since 3.0.0 */ - def kll_sketch_merge_bigint(left: Column, right: Column): Column = - Column.fn("kll_sketch_merge_bigint", left, right) + def bucket(numBuckets: Int, e: Column): Column = partitioning.bucket(numBuckets, e) + + ////////////////////////////////////////////////////////////////////////////////////////////// + // Predicates functions + ////////////////////////////////////////////////////////////////////////////////////////////// /** - * Merges two KLL float sketch buffers together into one. + * Returns `col2` if `col1` is null, or `col1` otherwise. * - * @param left - * The first KLL float sketch. A column that evaluates to a binary. - * @param right - * The second KLL float sketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param col1 + * The column to test for null. A column of any type. + * @param col2 + * The column to return when col1 is null. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column of the same type as the input. */ - def kll_sketch_merge_float(left: Column, right: Column): Column = - Column.fn("kll_sketch_merge_float", left, right) + def ifnull(col1: Column, col2: Column): Column = Column.fn("ifnull", col1, col2) /** - * Merges two KLL double sketch buffers together into one. + * Returns true if `col` is not null, or false otherwise. * - * @param left - * The first KLL double sketch. A column that evaluates to a binary. - * @param right - * The second KLL double sketch. A column that evaluates to a binary. - * @group sketch_funcs - * @since 4.1.0 + * @param col + * The column to check. A column of any type. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a binary. + * Returns a column that evaluates to a boolean. */ - def kll_sketch_merge_double(left: Column, right: Column): Column = - Column.fn("kll_sketch_merge_double", left, right) + def isnotnull(col: Column): Column = Column.fn("isnotnull", col) /** - * Extracts a quantile value from a KLL bigint sketch given an input rank value. The rank can be - * a single value or an array. + * Returns same result as the EQUAL(=) operator for non-null operands, but returns true if both + * are null, false if one of the them is null. * - * @param sketch - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @param rank - * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or - * an array. Must be a constant. - * @group sketch_funcs - * @since 4.1.0 + * @param col1 + * The first column to compare. A column of any type. + * @param col2 + * The second column to compare. A column of any type. + * @group predicate_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a long, or an array of longs when `rank` is an array. + * Returns a column that evaluates to a boolean. */ - def kll_sketch_get_quantile_bigint(sketch: Column, rank: Column): Column = - Column.fn("kll_sketch_get_quantile_bigint", sketch, rank) + def equal_null(col1: Column, col2: Column): Column = Column.fn("equal_null", col1, col2) /** - * Extracts a quantile value from a KLL float sketch given an input rank value. The rank can be - * a single value or an array. + * Returns null if `col1` equals to `col2`, or `col1` otherwise. * - * @param sketch - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @param rank - * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or - * an array. - * @group sketch_funcs - * @since 4.1.0 + * @param col1 + * The value to return if it is not equal to `col2`. A column of any type. + * @param col2 + * The value compared with `col1`. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a float, or an array of floats when `rank` is an array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_quantile_float(sketch: Column, rank: Column): Column = - Column.fn("kll_sketch_get_quantile_float", sketch, rank) + def nullif(col1: Column, col2: Column): Column = Column.fn("nullif", col1, col2) /** - * Extracts a quantile value from a KLL double sketch given an input rank value. The rank can be - * a single value or an array. + * Returns null if `col` is equal to zero, or `col` otherwise. * - * @param sketch - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @param rank - * The rank value(s) to extract (between 0.0 and 1.0). A column that evaluates to a numeric or - * an array. - * @group sketch_funcs - * @since 4.1.0 + * @param col + * The input value. A column that evaluates to a numeric. + * @group conditional_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `rank` is an - * array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_quantile_double(sketch: Column, rank: Column): Column = - Column.fn("kll_sketch_get_quantile_double", sketch, rank) + def nullifzero(col: Column): Column = Column.fn("nullifzero", col) /** - * Extracts a rank value from a KLL bigint sketch given an input quantile value. The quantile - * can be a single value or an array. + * Returns `col2` if `col1` is null, or `col1` otherwise. * - * @param sketch - * The KLL bigint sketch binary representation. A column that evaluates to a binary. - * @param quantile - * The quantile value(s) to lookup. A column that evaluates to an integral or an array. - * @group sketch_funcs - * @since 4.1.0 + * @param col1 + * The value to return if it is not null. A column of any type. + * @param col2 + * The value to return if `col1` is null. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an - * array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_rank_bigint(sketch: Column, quantile: Column): Column = - Column.fn("kll_sketch_get_rank_bigint", sketch, quantile) + def nvl(col1: Column, col2: Column): Column = Column.fn("nvl", col1, col2) /** - * Extracts a rank value from a KLL float sketch given an input quantile value. The quantile can - * be a single value or an array. + * Returns `col2` if `col1` is not null, or `col3` otherwise. * - * @param sketch - * The KLL float sketch binary representation. A column that evaluates to a binary. - * @param quantile - * The quantile value(s) to lookup. A column that evaluates to a numeric or an array. - * @group sketch_funcs - * @since 4.1.0 + * @param col1 + * The value that determines which branch to return. A column of any type. + * @param col2 + * The value to return if `col1` is not null. A column of any type. + * @param col3 + * The value to return if `col1` is null. A column of any type. + * @group conditional_funcs + * @since 3.5.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an - * array. + * Returns a column of the same type as the input. */ - def kll_sketch_get_rank_float(sketch: Column, quantile: Column): Column = - Column.fn("kll_sketch_get_rank_float", sketch, quantile) + def nvl2(col1: Column, col2: Column, col3: Column): Column = Column.fn("nvl2", col1, col2, col3) /** - * Extracts a rank value from a KLL double sketch given an input quantile value. The quantile - * can be a single value or an array. + * Returns zero if `col` is null, or `col` otherwise. * - * @param sketch - * The KLL double sketch binary representation. A column that evaluates to a binary. - * @param quantile - * The quantile value(s) to look up. A column that evaluates to a numeric or an array. Must be - * a constant. - * @group sketch_funcs - * @since 4.1.0 + * @param col + * The input value. A column that evaluates to a numeric. + * @group conditional_funcs + * @since 4.0.0 * @return - * Returns a column that evaluates to a double, or an array of doubles when `quantile` is an - * array. + * Returns a column of the same type as the input. + */ + def zeroifnull(col: Column): Column = Column.fn("zeroifnull", col) + + // scalastyle:off line.size.limit + // scalastyle:off parameter.number + + /* Use the following code to generate: + + (0 to 10).foreach { x => + val types = (1 to x).foldRight("RT")((i, s) => s"A$i, $s") + val typeSeq = "RT" +: (1 to x).map(i => s"A$i") + val typeTags = typeSeq.map(t => s"$t: TypeTag").mkString(", ") + val implicitTypeTags = typeSeq.map(t => s"implicitly[TypeTag[$t]]").mkString(", ") + println(s""" + |/** + | * Defines a Scala closure of $x arguments as user-defined function (UDF). + | * The data types are automatically inferred based on the Scala closure's + | * signature. By default the returned UDF is deterministic. To change it to + | * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`. + | * + | * @group udf_funcs + | * @since 1.3.0 + | */ + |def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = { + | SparkUserDefinedFunction(f, $implicitTypeTags) + |}""".stripMargin) + } + + (0 to 10).foreach { i => + val extTypeArgs = (0 to i).map(_ => "_").mkString(", ") + println(s""" + |/** + | * Defines a Java UDF$i instance as user-defined function (UDF). + | * The caller must specify the output data type, and there is no automatic input type coercion. + | * By default the returned UDF is deterministic. To change it to nondeterministic, call the + | * API `UserDefinedFunction.asNondeterministic()`. + | * + | * @group udf_funcs + | * @since 2.3.0 + | */ + |def udf(f: UDF$i[$extTypeArgs], returnType: DataType): UserDefinedFunction = { + | SparkUserDefinedFunction(ToScalaUDF(f), returnType, $i) + |}""".stripMargin) + } + */ - def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = - Column.fn("kll_sketch_get_rank_double", sketch, quantile) ////////////////////////////////////////////////////////////////////////////////////////////// - // Geospatial ST Functions + // ST geospatial functions ////////////////////////////////////////////////////////////////////////////////////////////// /** @@ -17333,194 +17258,30 @@ object functions { * geography or geometry. * @param srid * The new SRID of the geospatial value. A column that evaluates to an integer. - * @group st_funcs - * @since 4.1.0 - */ - def st_setsrid(geo: Column, srid: Int): Column = - Column.fn("st_setsrid", geo, lit(srid)) - - /** - * Returns the SRID of the input GEOGRAPHY or GEOMETRY value. - * - * @param geo - * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a - * geography or geometry. - * @group st_funcs - * @since 4.1.0 - * @return - * Returns a column that evaluates to an integer. - */ - def st_srid(geo: Column): Column = - Column.fn("st_srid", geo) - - ////////////////////////////////////////////////////////////////////////////////////////////// - // Vector Functions - ////////////////////////////////////////////////////////////////////////////////////////////// - - /** - * Returns the cosine similarity between two float vectors. - * @param left - * first vector column. A column that evaluates to an array. - * @param right - * second vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_cosine_similarity(left: Column, right: Column): Column = - Column.fn("vector_cosine_similarity", left, right) - - /** - * Returns the inner product (dot product) between two float vectors. - * @param left - * first vector column. A column that evaluates to an array. - * @param right - * second vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_inner_product(left: Column, right: Column): Column = - Column.fn("vector_inner_product", left, right) - - /** - * Returns the Euclidean (L2) distance between two float vectors. - * @param left - * first vector column. A column that evaluates to an array. - * @param right - * second vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_l2_distance(left: Column, right: Column): Column = - Column.fn("vector_l2_distance", left, right) - - /** - * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. - * @param vector - * input vector column. A column that evaluates to an array. - * @param degree - * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_norm(vector: Column, degree: Column): Column = - Column.fn("vector_norm", vector, degree) - - /** - * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). - * @param vector - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to a float. - */ - def vector_norm(vector: Column): Column = - Column.fn("vector_norm", vector) - - /** - * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. - * @param vector - * input vector column. A column that evaluates to an array. - * @param degree - * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to an array. - */ - def vector_normalize(vector: Column, degree: Column): Column = - Column.fn("vector_normalize", vector, degree) - - /** - * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). - * @param vector - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to an array. - */ - def vector_normalize(vector: Column): Column = - Column.fn("vector_normalize", vector) - - /** - * Aggregate function: returns the element-wise mean of float vectors in a group. - * @param col - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 - * @return - * Returns a column that evaluates to an array. + * @group st_funcs + * @since 4.1.0 */ - def vector_avg(col: Column): Column = Column.fn("vector_avg", col) + def st_setsrid(geo: Column, srid: Int): Column = + Column.fn("st_setsrid", geo, lit(srid)) /** - * Aggregate function: returns the element-wise sum of float vectors in a group. - * @param col - * input vector column. A column that evaluates to an array. - * @group vector_funcs - * @since 4.3.0 + * Returns the SRID of the input GEOGRAPHY or GEOMETRY value. + * + * @param geo + * A geospatial value, either a GEOGRAPHY or a GEOMETRY. A column that evaluates to a + * geography or geometry. + * @group st_funcs + * @since 4.1.0 * @return - * Returns a column that evaluates to an array. + * Returns a column that evaluates to an integer. */ - def vector_sum(col: Column): Column = Column.fn("vector_sum", col) + def st_srid(geo: Column): Column = + Column.fn("st_srid", geo) ////////////////////////////////////////////////////////////////////////////////////////////// - // UDF, UDAF and UDT + // Scala UDF functions ////////////////////////////////////////////////////////////////////////////////////////////// - // scalastyle:off line.size.limit - // scalastyle:off parameter.number - - /* Use the following code to generate: - - (0 to 10).foreach { x => - val types = (1 to x).foldRight("RT")((i, s) => s"A$i, $s") - val typeSeq = "RT" +: (1 to x).map(i => s"A$i") - val typeTags = typeSeq.map(t => s"$t: TypeTag").mkString(", ") - val implicitTypeTags = typeSeq.map(t => s"implicitly[TypeTag[$t]]").mkString(", ") - println(s""" - |/** - | * Defines a Scala closure of $x arguments as user-defined function (UDF). - | * The data types are automatically inferred based on the Scala closure's - | * signature. By default the returned UDF is deterministic. To change it to - | * nondeterministic, call the API `UserDefinedFunction.asNondeterministic()`. - | * - | * @group udf_funcs - | * @since 1.3.0 - | */ - |def udf[$typeTags](f: Function$x[$types]): UserDefinedFunction = { - | SparkUserDefinedFunction(f, $implicitTypeTags) - |}""".stripMargin) - } - - (0 to 10).foreach { i => - val extTypeArgs = (0 to i).map(_ => "_").mkString(", ") - println(s""" - |/** - | * Defines a Java UDF$i instance as user-defined function (UDF). - | * The caller must specify the output data type, and there is no automatic input type coercion. - | * By default the returned UDF is deterministic. To change it to nondeterministic, call the - | * API `UserDefinedFunction.asNondeterministic()`. - | * - | * @group udf_funcs - | * @since 2.3.0 - | */ - |def udf(f: UDF$i[$extTypeArgs], returnType: DataType): UserDefinedFunction = { - | SparkUserDefinedFunction(ToScalaUDF(f), returnType, $i) - |}""".stripMargin) - } - - */ - /** * Obtains a `UserDefinedFunction` that wraps the given `Aggregator` so that it may be used with * untyped Data Frames. @@ -17863,6 +17624,10 @@ object functions { implicitly[TypeTag[A10]]) } + ////////////////////////////////////////////////////////////////////////////////////////////// + // Java UDF functions + ////////////////////////////////////////////////////////////////////////////////////////////// + /** * Defines a Java UDF0 instance as user-defined function (UDF). The caller must specify the * output data type, and there is no automatic input type coercion. By default the returned UDF @@ -18070,6 +17835,21 @@ object functions { @scala.annotation.varargs def call_udf(udfName: String, cols: Column*): Column = call_function(udfName, cols: _*) + /** + * Call a SQL function. + * + * @param funcName + * function name that follows the SQL identifier syntax (can be quoted, can be qualified) + * @param cols + * the expression parameters of function + * @group normal_funcs + * @since 3.5.0 + */ + @scala.annotation.varargs + def call_function(funcName: String, cols: Column*): Column = { + Column(internal.UnresolvedFunction(funcName, cols.map(_.node), isUserDefinedFunction = true)) + } + /** * Unwrap UDT data type column into its underlying type. * @param column @@ -18104,4 +17884,177 @@ object functions { def wrap_udt(column: Column, udt: Column): Column = { Column.internalFn("wrap_udt", column, udt) } + + // ---------------------- Vector Functions ---------------------- + + /** + * Returns the cosine similarity between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_cosine_similarity(left: Column, right: Column): Column = + Column.fn("vector_cosine_similarity", left, right) + + /** + * Returns the inner product (dot product) between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_inner_product(left: Column, right: Column): Column = + Column.fn("vector_inner_product", left, right) + + /** + * Returns the Euclidean (L2) distance between two float vectors. + * @param left + * first vector column. A column that evaluates to an array. + * @param right + * second vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_l2_distance(left: Column, right: Column): Column = + Column.fn("vector_l2_distance", left, right) + + /** + * Returns the Lp norm of a float vector. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_norm(vector: Column, degree: Column): Column = + Column.fn("vector_norm", vector, degree) + + /** + * Returns the Lp norm of a float vector using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to a float. + */ + def vector_norm(vector: Column): Column = + Column.fn("vector_norm", vector) + + /** + * Normalizes a float vector to unit length. Degree defaults to 2.0 if unspecified. + * @param vector + * input vector column. A column that evaluates to an array. + * @param degree + * norm degree (1.0 for L1, 2.0 for L2, infinity norm). A column that evaluates to a float. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def vector_normalize(vector: Column, degree: Column): Column = + Column.fn("vector_normalize", vector, degree) + + /** + * Normalizes a float vector to unit length using degree 2.0 (Euclidean norm). + * @param vector + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def vector_normalize(vector: Column): Column = + Column.fn("vector_normalize", vector) + + /** + * Aggregate function: returns the element-wise mean of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def vector_avg(col: Column): Column = Column.fn("vector_avg", col) + + /** + * Aggregate function: returns the element-wise sum of float vectors in a group. + * @param col + * input vector column. A column that evaluates to an array. + * @group vector_funcs + * @since 4.3.0 + * @return + * Returns a column that evaluates to an array. + */ + def vector_sum(col: Column): Column = Column.fn("vector_sum", col) + + // scalastyle:off + // TODO(SPARK-45970): Use @static annotation so Java can access to those + // API in the same way. Once we land this fix, should deprecate + // functions.hours, days, months, years and bucket. + object partitioning { + // scalastyle:on + /** + * (Scala-specific) A transform for timestamps and dates to partition data into years. + * + * @group partition_transforms + * @since 4.0.0 + */ + def years(e: Column): Column = Column.internalFn("years", e) + + /** + * (Scala-specific) A transform for timestamps and dates to partition data into months. + * + * @group partition_transforms + * @since 4.0.0 + */ + def months(e: Column): Column = Column.internalFn("months", e) + + /** + * (Scala-specific) A transform for timestamps and dates to partition data into days. + * + * @group partition_transforms + * @since 4.0.0 + */ + def days(e: Column): Column = Column.internalFn("days", e) + + /** + * (Scala-specific) A transform for timestamps to partition data into hours. + * + * @group partition_transforms + * @since 4.0.0 + */ + def hours(e: Column): Column = Column.internalFn("hours", e) + + /** + * (Scala-specific) A transform for any type that partitions by a hash of the input column. + * + * @group partition_transforms + * @since 4.0.0 + */ + def bucket(numBuckets: Column, e: Column): Column = Column.internalFn("bucket", numBuckets, e) + + /** + * (Scala-specific) A transform for any type that partitions by a hash of the input column. + * + * @group partition_transforms + * @since 4.0.0 + */ + def bucket(numBuckets: Int, e: Column): Column = bucket(lit(numBuckets), e) + } } From 9025152347a7f1ac4eed45dfb5d4556f97606b23 Mon Sep 17 00:00:00 2001 From: Ruifeng Zheng Date: Wed, 2 Sep 2026 13:23:51 +0000 Subject: [PATCH 4/4] [SPARK-59170][SQL] Remove misleading function section headings --- python/pyspark/sql/functions/builtin.py | 37 ------------ .../org/apache/spark/sql/functions.scala | 60 +------------------ 2 files changed, 1 insertion(+), 96 deletions(-) diff --git a/python/pyspark/sql/functions/builtin.py b/python/pyspark/sql/functions/builtin.py index 54a55f91cd9fb..f22235f6d80df 100644 --- a/python/pyspark/sql/functions/builtin.py +++ b/python/pyspark/sql/functions/builtin.py @@ -111,7 +111,6 @@ # since it requires making every single overridden definition. # Public function groups are defined by pyspark.sql.functions.__all__ and mirrored in the API # reference. -# Section headings in this implementation file are only navigation aids. def _get_jvm_function(name: str, sc: "SparkContext") -> Callable: @@ -9216,9 +9215,6 @@ def factorial(col: "ColumnOrName") -> Column: return _invoke_function_over_columns("factorial", col) -# --------------- Window functions ------------------------ - - @_try_remote_functions def lag(col: "ColumnOrName", offset: int = 1, default: Optional[Any] = None) -> Column: """ @@ -9819,9 +9815,6 @@ def ntile(n: int) -> Column: return _invoke_function("ntile", int(_enum_to_value(n))) -# ---------------------- Date/Timestamp functions ------------------------------ - - @_try_remote_functions def curdate() -> Column: """ @@ -14554,9 +14547,6 @@ def to_timestamp_ntz( return _invoke_function_over_columns("to_timestamp_ntz", timestamp) -# ---------------------------- misc functions ---------------------------------- - - @_try_remote_functions def current_catalog() -> Column: """Returns the current catalog. @@ -15240,9 +15230,6 @@ def raise_error(errMsg: Union[Column, str]) -> Column: return _invoke_function_over_columns("raise_error", lit(errMsg)) -# ---------------------- String/Binary functions ------------------------------ - - @_try_remote_functions def upper(col: "ColumnOrName") -> Column: """ @@ -19942,9 +19929,6 @@ def quote(col: "ColumnOrName") -> Column: return _invoke_function_over_columns("quote", col) -# ---------------------- Collection functions ------------------------------ - - @overload def create_map(*cols: "ColumnOrName") -> Column: ... @@ -26851,9 +26835,6 @@ def str_to_map( return _invoke_function_over_columns("str_to_map", text, pairDelim, keyValueDelim) -# ---------------------- Partition transform functions -------------------------------- - - @_try_remote_functions def years(col: "ColumnOrName") -> Column: """ @@ -28913,9 +28894,6 @@ def bucket(numBuckets: Union[Column, int], col: "ColumnOrName") -> Column: return partitioning.bucket(numBuckets, col) -# Geospatial ST Functions - - @_try_remote_functions def st_asbinary(geo: "ColumnOrName", endianness: Optional["ColumnOrName"] = None) -> Column: """Returns the input GEOGRAPHY or GEOMETRY value in WKB format. @@ -29083,9 +29061,6 @@ def st_srid(geo: "ColumnOrName") -> Column: return _invoke_function_over_columns("st_srid", geo) -# Call Functions - - @_try_remote_functions def call_udf(udfName: str, *cols: "ColumnOrName") -> Column: """ @@ -29372,9 +29347,6 @@ def wrap_udt(col: "ColumnOrName", udt: "Union[UserDefinedType, Column]") -> Colu return _invoke_function("wrap_udt", _to_java_column(col), _to_java_column(udt_col)) -# ---------------------- Datasketch functions ------------------------------ - - @_try_remote_functions def hll_sketch_agg( col: "ColumnOrName", @@ -31918,9 +31890,6 @@ def tuple_union_theta_integer( return _invoke_function_over_columns(fn, col1, col2, _lgNomEntries, _mode) -# ---------------------- Predicates functions ------------------------------ - - @_try_remote_functions def ifnull(col1: "ColumnOrName", col2: "ColumnOrName") -> Column: """ @@ -33465,9 +33434,6 @@ def bitmap_xor_agg(col: "ColumnOrName") -> Column: return _invoke_function_over_columns("bitmap_xor_agg", col) -# ---------------------------- User Defined Function ---------------------------------- - - def udaf(agg: "Aggregator") -> "UserDefinedFunctionLike": """Turn an :class:`~pyspark.sql.aggregator.Aggregator` instance into a callable usable in ``groupBy().agg(...)`` (and as a window function), the Python counterpart of Scala's @@ -34041,9 +34007,6 @@ def arrow_udtf( return _create_pyarrow_udtf(cls=cls, returnType=returnType) -# ---------------------- Vector Functions ---------------------- - - @_try_remote_functions def vector_cosine_similarity(left: "ColumnOrName", right: "ColumnOrName") -> Column: """Returns the cosine similarity between two float vectors. diff --git a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala index efa69f4154ce6..589edde9c171c 100644 --- a/sql/api/src/main/scala/org/apache/spark/sql/functions.scala +++ b/sql/api/src/main/scala/org/apache/spark/sql/functions.scala @@ -89,7 +89,7 @@ object functions { // scalastyle:on // Function groups are defined by the @group tags above each function and the corresponding - // @groupname declarations. Section headings in this implementation file are navigation aids. + // @groupname declarations. /** * Returns a [[Column]] based on the given column name. @@ -174,10 +174,6 @@ object functions { } } - ////////////////////////////////////////////////////////////////////////////////////////////// - // Sort functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Returns a sort expression based on ascending order of the column. * {{{ @@ -248,10 +244,6 @@ object functions { */ def desc_nulls_last(columnName: String): Column = Column(columnName).desc_nulls_last - ////////////////////////////////////////////////////////////////////////////////////////////// - // Aggregate functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * @group agg_funcs * @since 1.3.0 @@ -3832,10 +3824,6 @@ object functions { */ def bit_xor(e: Column): Column = Column.fn("bit_xor", e) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Window functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Window function: computes the differences between consecutive cumulative counter values in a * time series, thereby converting the counter from the cumulative to the delta format. @@ -4246,10 +4234,6 @@ object functions { */ def row_number(): Column = Column.fn("row_number") - ////////////////////////////////////////////////////////////////////////////////////////////// - // Non-aggregate functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Creates a new array column. The input columns must all have the same data type. * @@ -4911,10 +4895,6 @@ object functions { */ def expr(expr: String): Column = Column(internal.SqlExpression(expr)) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Math Functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Computes the absolute value of a numeric value. * @@ -6502,10 +6482,6 @@ object functions { def width_bucket(v: Column, min: Column, max: Column, numBucket: Column): Column = Column.fn("width_bucket", v, min, max, numBucket) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Misc functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Returns the current catalog. * @@ -7468,10 +7444,6 @@ object functions { */ def bitmap_xor_agg(col: Column): Column = Column.fn("bitmap_xor_agg", col) - ////////////////////////////////////////////////////////////////////////////////////////////// - // String functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Computes the numeric value of the first character of the string column, and returns the * result as an int column. @@ -9515,10 +9487,6 @@ object functions { */ def quote(str: Column): Column = Column.fn("quote", str) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Datasketch functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Returns the estimated number of unique values given the binary representation of a * Datasketches HllSketch. @@ -11518,10 +11486,6 @@ object functions { def kll_sketch_get_rank_double(sketch: Column, quantile: Column): Column = Column.fn("kll_sketch_get_rank_double", sketch, quantile) - ////////////////////////////////////////////////////////////////////////////////////////////// - // DateTime functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Returns the date that is `numMonths` after `startDate`. * @@ -13232,10 +13196,6 @@ object functions { def dayname(timeExp: Column): Column = Column.fn("dayname", timeExp) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Collection functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Returns true if the array contains `value`, false if not. Returns null if the array or * `value` is null, or if `value` is not found and the array contains a null element. @@ -16979,10 +16939,6 @@ object functions { */ def bucket(numBuckets: Int, e: Column): Column = partitioning.bucket(numBuckets, e) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Predicates functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Returns `col2` if `col1` is null, or `col1` otherwise. * @@ -17136,10 +17092,6 @@ object functions { */ - ////////////////////////////////////////////////////////////////////////////////////////////// - // ST geospatial functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Returns the input GEOGRAPHY or GEOMETRY value in WKB format. * @@ -17278,10 +17230,6 @@ object functions { def st_srid(geo: Column): Column = Column.fn("st_srid", geo) - ////////////////////////////////////////////////////////////////////////////////////////////// - // Scala UDF functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Obtains a `UserDefinedFunction` that wraps the given `Aggregator` so that it may be used with * untyped Data Frames. @@ -17624,10 +17572,6 @@ object functions { implicitly[TypeTag[A10]]) } - ////////////////////////////////////////////////////////////////////////////////////////////// - // Java UDF functions - ////////////////////////////////////////////////////////////////////////////////////////////// - /** * Defines a Java UDF0 instance as user-defined function (UDF). The caller must specify the * output data type, and there is no automatic input type coercion. By default the returned UDF @@ -17885,8 +17829,6 @@ object functions { Column.internalFn("wrap_udt", column, udt) } - // ---------------------- Vector Functions ---------------------- - /** * Returns the cosine similarity between two float vectors. * @param left