Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
48 changes: 48 additions & 0 deletions pyiceberg/conversions.py
Original file line number Diff line number Diff line change
Expand Up @@ -76,11 +76,15 @@
time_str_to_micros,
time_to_micros,
timestamp_to_micros,
timestamp_to_nanos,
timestamptz_to_micros,
timestamptz_to_nanos,
to_human_day,
to_human_time,
to_human_timestamp,
to_human_timestamp_ns,
to_human_timestamptz,
to_human_timestamptz_ns,
)
from pyiceberg.utils.decimal import decimal_to_bytes, unscaled_to_decimal

Expand Down Expand Up @@ -487,6 +491,22 @@ def _(_: TimestamptzType, val: int | datetime) -> str:
return to_human_timestamptz(val)


@to_json.register(TimestampNanoType)
def _(_: TimestampNanoType, val: int | datetime) -> str:
"""Python datetime (without timezone) or nanoseconds since epoch serializes into an ISO8601 timestamp."""
if isinstance(val, datetime):
val = datetime_to_nanos(val)
return to_human_timestamp_ns(val)


@to_json.register(TimestamptzNanoType)
def _(_: TimestamptzNanoType, val: int | datetime) -> str:
"""Python datetime (with timezone) or nanoseconds since epoch serializes into an ISO8601 timestamp."""
if isinstance(val, datetime):
val = datetime_to_nanos(val)
return to_human_timestamptz_ns(val)


@to_json.register(FloatType)
@to_json.register(DoubleType)
def _(_: FloatType | DoubleType, val: float) -> float:
Expand Down Expand Up @@ -641,6 +661,34 @@ def _(_: TimestamptzType, val: str | int | datetime) -> datetime:
return val


@from_json.register(TimestampNanoType)
def _(_: TimestampNanoType, val: str | int | datetime) -> int:
"""JSON ISO8601 string into nanoseconds since epoch.

Python datetime cannot hold nanoseconds, so the value stays an int.
"""
if isinstance(val, str):
return timestamp_to_nanos(val)
elif isinstance(val, datetime):
return datetime_to_nanos(val)
else:
return val


@from_json.register(TimestamptzNanoType)
def _(_: TimestamptzNanoType, val: str | int | datetime) -> int:
"""JSON ISO8601 string into nanoseconds since epoch.

Python datetime cannot hold nanoseconds, so the value stays an int.
"""
if isinstance(val, str):
return timestamptz_to_nanos(val)
elif isinstance(val, datetime):
return datetime_to_nanos(val)
else:
return val


@from_json.register(FloatType)
@from_json.register(DoubleType)
def _(_: FloatType | DoubleType, val: float) -> float:
Expand Down
14 changes: 14 additions & 0 deletions pyiceberg/utils/datetime.py
Original file line number Diff line number Diff line change
Expand Up @@ -224,6 +224,20 @@ def to_human_timestamp(timestamp_micros: int) -> str:
return (EPOCH_TIMESTAMP + timedelta(microseconds=timestamp_micros)).isoformat()


def to_human_timestamp_ns(timestamp_nanos: int) -> str:
"""Convert a TimestampNanoType value to human string."""
seconds, nanos = divmod(timestamp_nanos, 1_000_000_000)
timestamp = EPOCH_TIMESTAMP + timedelta(seconds=seconds)
return f"{timestamp.isoformat(timespec='seconds')}.{nanos:09d}"


def to_human_timestamptz_ns(timestamp_nanos: int) -> str:
"""Convert a TimestamptzNanoType value to human string."""
seconds, nanos = divmod(timestamp_nanos, 1_000_000_000)
timestamp = EPOCH_TIMESTAMPTZ + timedelta(seconds=seconds)
return f"{timestamp.replace(tzinfo=None).isoformat(timespec='seconds')}.{nanos:09d}+00:00"


def micros_to_hours(micros: int) -> int:
"""Convert a timestamp in microseconds to hours from 1970-01-01T00:00."""
return micros // 3_600_000_000
Expand Down
16 changes: 16 additions & 0 deletions tests/test_conversions.py
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,14 @@ def test_datetime_obj_to_bytes(primitive_type: PrimitiveType, value: datetime |
(TimeType(), time(22, 31, 8, 123456), "22:31:08.123456"),
(TimestampType(), datetime(2017, 11, 16, 22, 31, 8, 123456), "2017-11-16T22:31:08.123456"),
(TimestamptzType(), datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC), "2017-11-16T22:31:08.123456+00:00"),
(TimestampNanoType(), 1510871468123456789, "2017-11-16T22:31:08.123456789"),
(TimestamptzNanoType(), 1510871468123456789, "2017-11-16T22:31:08.123456789+00:00"),
(TimestampNanoType(), datetime(2017, 11, 16, 22, 31, 8, 123456), "2017-11-16T22:31:08.123456000"),
(
TimestamptzNanoType(),
datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC),
"2017-11-16T22:31:08.123456000+00:00",
),
(StringType(), "iceberg", "iceberg"),
(BinaryType(), b"\x01\x02\x03\xff", "010203ff"),
(FixedType(4), b"\x01\x02\x03\xff", "010203ff"),
Expand All @@ -635,6 +643,8 @@ def test_json_single_serialization(primitive_type: PrimitiveType, value: Any, ex
(TimeType(), time(22, 31, 8, 123456)),
(TimestampType(), datetime(2017, 11, 16, 22, 31, 8, 123456)),
(TimestamptzType(), datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC)),
(TimestampNanoType(), 1510871468123456789),
(TimestamptzNanoType(), 1510871468123456789),
(StringType(), "iceberg"),
(BinaryType(), b"\x01\x02\x03\xff"),
(FixedType(4), b"\x01\x02\x03\xff"),
Expand All @@ -653,6 +663,12 @@ def test_json_serialize_roundtrip(primitive_type: PrimitiveType, value: Any) ->
"2017-11-16T22:31:08.123456789+00:00",
datetime(2017, 11, 16, 22, 31, 8, 123456, tzinfo=UTC),
),
(TimestampNanoType(), "2017-11-16T22:31:08.123456789", 1510871468123456789),
(
TimestamptzNanoType(),
"2017-11-16T22:31:08.123456789+00:00",
1510871468123456789,
),
],
)
def test_json_single_deserialization_with_nano_precision(
Expand Down
17 changes: 17 additions & 0 deletions tests/test_types.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,9 @@
PrimitiveType,
StringType,
StructType,
TimestampNanoType,
TimestampType,
TimestamptzNanoType,
TimestamptzType,
TimeType,
UUIDType,
Expand Down Expand Up @@ -947,3 +949,18 @@ def test_decimal_scale_validation() -> None:

with pytest.raises(ValidationError, match="Decimal scale must be between 0 and the precision"):
DecimalType(5, 10)


@pytest.mark.parametrize(
"field_type, expected_json",
[
(TimestampNanoType(), "2017-11-16T22:31:08.123456789"),
(TimestamptzNanoType(), "2017-11-16T22:31:08.123456789+00:00"),
],
)
def test_nested_field_nanosecond_defaults(field_type: PrimitiveType, expected_json: str) -> None:
"""Nanosecond defaults load as nanoseconds since epoch and serialize back without losing precision."""
field = NestedField(1, "ts", field_type, initial_default=expected_json)

assert field.initial_default == 1510871468123456789
assert field.model_dump()["initial-default"] == expected_json
Loading