Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 9 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,12 @@ part of your geospatial project.

# Version Changes

## 3.1.7
### Bug fix
- Warn (or raise the new `DbfNumericDataLoss` in strict mode) when a number is too wide for its
"N" or "F" field and has to be truncated, instead of silently writing a different number.
`DbfStringDataLoss` and `DbfNumericDataLoss` now share a `DbfDataLoss` base class.

## 3.1.6
### Feature
- Encodings can now be read from .cpg files (and optionally written to them).
Expand Down Expand Up @@ -1073,7 +1079,9 @@ Numeric fields are created using the 'N' type (or the 'F' type, which is exactly
By default the fourth decimal argument is set to zero, essentially creating an integer field.
To store floats you must set the decimal argument to the precision of your choice.
To store very large numbers you must increase the field length size to the total number of digits
(including comma and minus).
(including comma and minus). A number that is wider than its field has to be truncated to keep the
record layout intact, which writes a different number, so PyShp warns about this (and raises
`DbfNumericDataLoss` in strict mode).


>>> w = shapefile.Writer('tests/shapefiles/test/dtype')
Expand Down
4 changes: 4 additions & 0 deletions changelog.txt
Original file line number Diff line number Diff line change
@@ -1,3 +1,7 @@
VERSION 3.1.7
2026-08-06
* Warn (or raise the new DbfNumericDataLoss in strict mode) when a number is too wide for its "N" or "F" field and has to be truncated, instead of silently writing a different number.

VERSION 3.1.6
2026-07-25
* Encodings can now be read from .cpg files (and optionally written to them).
Expand Down
51 changes: 44 additions & 7 deletions src/shapefile.py
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,15 @@ class PossibleDataLoss(Warning):
pass


class DbfStringDataLoss(ValueError):
class DbfDataLoss(ValueError):
pass


class DbfStringDataLoss(DbfDataLoss):
pass


class DbfNumericDataLoss(DbfDataLoss):
pass


Expand Down Expand Up @@ -457,6 +465,35 @@ def check_and_trim(decoded_pad_bytes: dict[str, bytes]) -> None:
return padded, trimmed


def _pack_dbf_number(
formatted: str,
size: int,
value: RecordValue,
field_name: str,
strict: bool = True,
) -> str:
"""Right justifies an already formatted number in a field of width size.

A truncated number is a different number, not a lossy version of the data
like a truncated string is, so warn about it (or raise in strict mode),
as the "C" and "M" field paths do.
"""
if len(formatted) > size:
msg = (
f"Formatted: {formatted} of data: {value!r} needs {len(formatted)} bytes, "
f"and was truncated to: {formatted[:size]} "
f"to fit within the {size=} bytes of field: {field_name!r}, "
"changing the number that is written. "
"To avoid data loss, make a new Writer or dbfWriter and call .field "
"with a bigger size (and if applicable, a smaller decimal). "
)
if strict:
raise DbfNumericDataLoss(msg)
warnings.warn(msg, category=PossibleDataLoss)

return formatted[:size].rjust(size)


def _try_to_decode_dbf_name_or_text_field(
b: bytes,
pad_bytes: bytes, # Pad bytes will be trimmed from the RHS (end) of b.
Expand Down Expand Up @@ -4451,14 +4488,14 @@ def _record(self, record: list[RecordValue]) -> None:
except ValueError:
# forcing directly to int failed, so was probably a float.
num_val = int(float(cast(float, value)))
str_val = format(num_val, "d")[:size].rjust(
size
) # caps the size if exceeds the field size
str_val = _pack_dbf_number(
format(num_val, "d"), size, value, fieldName, self.strict
)
else:
f_val = float(cast(float, value))
str_val = format(f_val, f".{deci}f")[:size].rjust(
size
) # caps the size if exceeds the field size
str_val = _pack_dbf_number(
format(f_val, f".{deci}f"), size, value, fieldName, self.strict
)
elif fieldType == "D":
# date: 8 bytes - date stored as a string in the format YYYYMMDD.
if isinstance(value, list) and len(value) == 3:
Expand Down
4 changes: 3 additions & 1 deletion tests/hypothesis_tests.py
Original file line number Diff line number Diff line change
Expand Up @@ -881,7 +881,9 @@ def _write_fields_and_records_to_strict(w, fields, records):
]
try:
w.record(*rec_list)
except shp.DbfStringDataLoss:
except shp.DbfDataLoss:
# Numbers too wide for their field are rejected in strict mode too,
# e.g. round(9999999.96875, 1) needs 10 chars in an N field of size 9.
written_records.append(None)
else:
written_records.append(rec_list)
Expand Down
95 changes: 95 additions & 0 deletions tests/test_shapefile.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
import os.path
from pathlib import Path
import shutil
import warnings

# third party imports
import pytest
Expand Down Expand Up @@ -2068,6 +2069,100 @@ def test_encode_dbf_field_padding_bytes_errors(value,encoded_len,codec,errors):
w.record(value)
w.close()


def _write_one_field_record(field_type, size, decimal, value, strict):
"""Returns the raw bytes of the single field, and the value read back."""
stream = io.BytesIO()
w = shapefile.DbfWriter(dbf=stream, strict=strict)
w.field("V", field_type, size=size, decimal=decimal)
w.record(value)
w.close()

raw = stream.getvalue()
# 32 byte header + 32 bytes per field + terminator + the record's deletion flag.
start = 32 + 32 + 1 + 1
with shapefile.DbfReader(dbf=io.BytesIO(raw)) as r:
# DeletionFlag is r.fields[0].
written_size = r.fields[1][2]
return raw[start:start + written_size], r.record(0)[0]

# Numbers that do not fit their field, with the corrupted bytes and the
# different number that PyShp writes in their place.
NUMERIC_FIELD_OVERFLOWS = [
("N", 5, 0, 123456789, b"12345", 12345),
("N", 5, 0, -123456, b"-1234", -1234),
("N", 2, 0, -999, b"-9", -9),
("N", 1, 0, -5, b"-", None), # nothing but the sign is left
("N", 3, 0, 10 ** 30, b"100", 100),
("N", 6, 2, 12345.67, b"12345.", 12345.0), # ends in a bare decimal point
("N", 5, 2, 12345.67, b"12345", 12345.0),
("N", 3, 1, -1.25, b"-1.", -1.0),
("N", 9, 1, 9999999.96875, b"10000000.", 10000000.0), # rounding carries a digit
("F", 5, 0, 123456789, b"12345", 12345),
("F", 6, 2, 12345.67, b"12345.", 12345.0),
("F", 4, 2, -1.25, b"-1.2", -1.2),
]

# The same fields and values, sized so that nothing has to be truncated.
NUMERIC_FIELDS_THAT_FIT = [
("N", 5, 0, 12345),
("N", 5, 0, -1234),
("N", 1, 0, 7),
("N", 8, 2, 12345.67),
("N", 10, 3, 1.5),
("F", 9, 1, 9999999.9),
("F", 6, 2, -1.25),
]

@pytest.mark.parametrize("field_type,size,decimal,value,expected_bytes,read_back", NUMERIC_FIELD_OVERFLOWS)
def test_numeric_field_overflow_warns_and_corrupts_the_value(
field_type, size, decimal, value, expected_bytes, read_back
):
with pytest.warns(shapefile.PossibleDataLoss):
written, actual = _write_one_field_record(field_type, size, decimal, value, strict=False)

assert written == expected_bytes
assert actual == read_back
assert actual != value

@pytest.mark.parametrize("field_type,size,decimal,value,expected_bytes,read_back", NUMERIC_FIELD_OVERFLOWS)
def test_numeric_field_overflow_raises_in_strict_mode(
field_type, size, decimal, value, expected_bytes, read_back
):
with pytest.raises(shapefile.DbfNumericDataLoss):
_write_one_field_record(field_type, size, decimal, value, strict=True)

@pytest.mark.parametrize("strict", [False, True])
@pytest.mark.parametrize("field_type,size,decimal,value", NUMERIC_FIELDS_THAT_FIT)
def test_numeric_field_that_fits_round_trips_without_warning(field_type, size, decimal, value, strict):
with warnings.catch_warnings():
warnings.simplefilter("error", shapefile.PossibleDataLoss)
written, actual = _write_one_field_record(field_type, size, decimal, value, strict)

assert len(written) == size
assert actual == value

@pytest.mark.parametrize("strict", [False, True])
@pytest.mark.parametrize("field_type,size,value,expected_bytes", [
("D", 4, datetime.date(2026, 8, 6), b"20260806"),
("D", 20, datetime.date(2026, 8, 6), b"20260806"),
("L", 4, True, b"T"),
("L", 20, False, b"F"),
])
def test_date_and_logical_fields_cannot_overflow(field_type, size, value, expected_bytes, strict):
# Field.from_unchecked forces "D" to 8 bytes and "L" to 1, so unlike the
# "N" and "F" fields, a mis-sized field cannot truncate the value.
with warnings.catch_warnings():
warnings.simplefilter("error", shapefile.PossibleDataLoss)
written, _actual = _write_one_field_record(field_type, size, 0, value, strict)

assert written == expected_bytes

def test_dbf_data_loss_exceptions_share_a_base_class():
assert issubclass(shapefile.DbfStringDataLoss, shapefile.DbfDataLoss)
assert issubclass(shapefile.DbfNumericDataLoss, shapefile.DbfDataLoss)
assert issubclass(shapefile.DbfDataLoss, ValueError)

LONG_FIELD_NAMES = [
("ÀÀÀÀ०", 8, "utf-8", "strict"), # Encoded bytes are corrupted if truncated to 10 bytes
]
Expand Down