Skip to content

GH-51760: [C++][Python] Emit decimal points for integral floats #51767

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
gitedmond wants to merge 1 commit into
base: main
Choose a base branch
from
+61 −30
Open

GH-51760: [C++][Python] Emit decimal points for integral floats #51767

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 6 additions & 3 deletions cpp/src/arrow/compute/kernels/scalar_cast_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3571,9 +3571,12 @@ TEST(Cast, IntToString) {
TEST(Cast, FloatingToString) {
for (auto float_type : {float16(), float32(), float64()}) {
for (auto string_type : {utf8(), utf8_view(), large_utf8()}) {
CheckCast(ArrayFromJSON(float_type, "[0.0, -0.0, 1.5, -Inf, Inf, NaN, null]"),
ArrayFromJSON(string_type,
R"(["0", "-0", "1.5", "-inf", "inf", "nan", null])"));
CheckCast(
ArrayFromJSON(float_type,
"[0.0, -0.0, 20.0, -20.0, 1.5, -Inf, Inf, NaN, null]"),
ArrayFromJSON(
string_type,
R"(["0.0", "-0.0", "20.0", "-20.0", "1.5", "-inf", "inf", "nan", null])"));
}
}
}
Expand Down
24 changes: 12 additions & 12 deletions cpp/src/arrow/pretty_print_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,18 +147,18 @@ TEST_F(TestPrettyPrint, PrimitiveType) {

std::vector<double> values2 = {0., 1., 2., 3., 4.};
static const char* ex2 = R"expected([
0,
1,
0.0,
1.0,
null,
3,
3.0,
null
])expected";
CheckPrimitive<DoubleType, double>({0, 10}, is_valid, values2, ex2);
static const char* ex2_in2 = R"expected( [
0,
1,
0.0,
1.0,
null,
3,
3.0,
null
])expected";
CheckPrimitive<DoubleType, double>({2, 10}, is_valid, values2, ex2_in2);
Expand Down Expand Up @@ -337,16 +337,16 @@ TEST_F(TestPrettyPrint, UInt64) {
TEST_F(TestPrettyPrint, HalfFloat) {
static const char* expected = R"expected([
-inf,
-1234,
-0,
0,
1,
-1234.0,
-0.0,
0.0,
1.0,
1.2001953125,
2.5,
3.9921875,
4.125,
10000,
12344,
10000.0,
12344.0,
inf,
nan,
null
Expand Down
3 changes: 2 additions & 1 deletion cpp/src/arrow/scalar_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,7 +150,8 @@ TEST(TestBooleanScalar, Cast) {
ARROW_SCOPED_TRACE("to type: ", to_type->ToString());
ASSERT_OK_AND_ASSIGN(auto casted, Cast(scalar, to_type));
ASSERT_EQ(casted.scalar()->type->id(), to_type->id());
ASSERT_EQ(casted.scalar()->ToString(), std::to_string(b));
ASSERT_EQ(casted.scalar()->ToString(),
std::to_string(b) + (is_floating(*to_type) ? ".0" : ""));
}

// String type.
Expand Down
6 changes: 4 additions & 2 deletions cpp/src/arrow/util/formatting.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,10 @@ const char digit_pairs[] =

struct FloatToStringFormatter::Impl {
Impl()
: converter_(DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN, "inf", "nan",
'e', -6, 10, 6, 0) {}
: converter_(DoubleToStringConverter::EMIT_POSITIVE_EXPONENT_SIGN |
DoubleToStringConverter::EMIT_TRAILING_DECIMAL_POINT |
DoubleToStringConverter::EMIT_TRAILING_ZERO_AFTER_POINT,
"inf", "nan", 'e', -6, 10, 6, 0) {}

Impl(int flags, const char* inf_symbol, const char* nan_symbol, char exp_character,
int decimal_in_shortest_low, int decimal_in_shortest_high,
Expand Down
30 changes: 18 additions & 12 deletions cpp/src/arrow/util/formatting_util_test.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -246,12 +246,14 @@ TEST(Formatting, Int64) {
TEST(Formatting, Float) {
StringFormatter<FloatType> formatter;

AssertFormatting(formatter, 0.0f, "0");
AssertFormatting(formatter, -0.0f, "-0");
AssertFormatting(formatter, 0.0f, "0.0");
AssertFormatting(formatter, -0.0f, "-0.0");
AssertFormatting(formatter, 20.0f, "20.0");
AssertFormatting(formatter, -20.0f, "-20.0");
AssertFormatting(formatter, 1.5f, "1.5");
AssertFormatting(formatter, 0.0001f, "0.0001");
AssertFormatting(formatter, 1234.567f, "1234.567");
AssertFormatting(formatter, 1e9f, "1000000000");
AssertFormatting(formatter, 1e9f, "1000000000.0");
AssertFormatting(formatter, 1e10f, "1e+10");
AssertFormatting(formatter, 1e20f, "1e+20");
AssertFormatting(formatter, 1e-6f, "0.000001");
Expand All @@ -266,12 +268,14 @@ TEST(Formatting, Float) {
TEST(Formatting, Double) {
StringFormatter<DoubleType> formatter;

AssertFormatting(formatter, 0.0, "0");
AssertFormatting(formatter, -0.0, "-0");
AssertFormatting(formatter, 0.0, "0.0");
AssertFormatting(formatter, -0.0, "-0.0");
AssertFormatting(formatter, 20.0, "20.0");
AssertFormatting(formatter, -20.0, "-20.0");
AssertFormatting(formatter, 1.5, "1.5");
AssertFormatting(formatter, 0.0001, "0.0001");
AssertFormatting(formatter, 1234.567, "1234.567");
AssertFormatting(formatter, 1e9, "1000000000");
AssertFormatting(formatter, 1e9, "1000000000.0");
AssertFormatting(formatter, 1e10, "1e+10");
AssertFormatting(formatter, 1e20, "1e+20");
AssertFormatting(formatter, 1e-6, "0.000001");
Expand All @@ -286,21 +290,23 @@ TEST(Formatting, Double) {
TEST(Formatting, HalfFloat) {
StringFormatter<HalfFloatType> formatter;

AssertFormatting(formatter, Float16(0.0f).bits(), "0");
AssertFormatting(formatter, Float16(-0.0f).bits(), "-0");
AssertFormatting(formatter, Float16(0.0f).bits(), "0.0");
AssertFormatting(formatter, Float16(-0.0f).bits(), "-0.0");
AssertFormatting(formatter, Float16(20.0f).bits(), "20.0");
AssertFormatting(formatter, Float16(-20.0f).bits(), "-20.0");
AssertFormatting(formatter, Float16(1.5f).bits(), "1.5");

// Slightly adapted from values present here
// https://blogs.mathworks.com/cleve/2017/05/08/half-precision-16-bit-floating-point-arithmetic/
AssertFormatting(formatter, 0x3c00, "1");
AssertFormatting(formatter, 0x3c00, "1.0");
AssertFormatting(formatter, 0x3c01, "1.0009765625");
AssertFormatting(formatter, 0x0400, "0.00006103515625");
AssertFormatting(formatter, 0x0001, "5.960464477539063e-8");

// Can't avoid loss of precision here.
AssertFormatting(formatter, Float16(1234.567f).bits(), "1235");
AssertFormatting(formatter, Float16(1e3f).bits(), "1000");
AssertFormatting(formatter, Float16(1e4f).bits(), "10000");
AssertFormatting(formatter, Float16(1234.567f).bits(), "1235.0");
AssertFormatting(formatter, Float16(1e3f).bits(), "1000.0");
AssertFormatting(formatter, Float16(1e4f).bits(), "10000.0");
AssertFormatting(formatter, Float16(1e10f).bits(), "inf");
AssertFormatting(formatter, Float16(1e15f).bits(), "inf");

Expand Down
19 changes: 19 additions & 0 deletions python/pyarrow/tests/test_dataset.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3503,6 +3503,25 @@ def test_csv_format(tempdir, dataset_reader):
assert result.equals(table)


@pytest.mark.parametrize("float_type", [pa.float32(), pa.float64()])
def test_csv_integral_floats_preserve_inferred_type(
tempdir, float_type, dataset_reader):
tables = [
pa.table({'x': pa.array([20.0, 21.0], type=float_type), 'i': [20, 21]}),
pa.table({'x': pa.array([20.5, 21.25], type=float_type), 'i': [22, 23]}),
]
paths = [str(tempdir / f't{i}.csv') for i in range(len(tables))]
for table, path in zip(tables, paths):
pyarrow.csv.write_csv(table, path)

assert pathlib.Path(paths[0]).read_text() == '"x","i"\n20.0,20\n21.0,21\n'
expected = pa.table({'x': [20.0, 21.0, 20.5, 21.25],
'i': [20, 21, 22, 23]})
dataset = ds.dataset(paths, format='csv')
assert dataset.schema == expected.schema
assert dataset_reader.to_table(dataset).equals(expected)


@pytest.mark.pandas
@pytest.mark.parametrize("compression", [
"bz2",
Expand Down
Loading