Skip to content

Commit f2f373f

Browse files
zoobazooba
committed
Issue python#23152: Implement _Py_fstat() to support files larger than 2 GB on Windows.
fstat() may fail with EOVERFLOW on files larger than 2 GB because the file size type is an signed 32-bit integer.
1 parent 18d1924 commit f2f373f

12 files changed

Lines changed: 242 additions & 180 deletions

File tree

‎Include/fileutils.h‎

Lines changed: 30 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,11 +21,40 @@ PyAPI_FUNC(int) _Py_wstat(
2121
struct stat *buf);
2222
#endif
2323

24+
#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
25+
26+
#ifdef MS_WINDOWS
27+
struct _Py_stat_struct {
28+
unsigned long st_dev;
29+
__int64 st_ino;
30+
unsigned short st_mode;
31+
int st_nlink;
32+
int st_uid;
33+
int st_gid;
34+
unsigned long st_rdev;
35+
__int64 st_size;
36+
time_t st_atime;
37+
int st_atime_nsec;
38+
time_t st_mtime;
39+
int st_mtime_nsec;
40+
time_t st_ctime;
41+
int st_ctime_nsec;
42+
unsigned long st_file_attributes;
43+
};
44+
#else
45+
# define _Py_stat_struct stat
46+
#endif
47+
48+
PyAPI_FUNC(int) _Py_fstat(
49+
int fd,
50+
struct _Py_stat_struct *stat);
51+
#endif /* HAVE_FSTAT || MS_WINDOWS */
52+
2453
#ifdef HAVE_STAT
2554
PyAPI_FUNC(int) _Py_stat(
2655
PyObject *path,
2756
struct stat *statbuf);
28-
#endif
57+
#endif /* HAVE_STAT */
2958

3059
#ifndef Py_LIMITED_API
3160
PyAPI_FUNC(int) _Py_open(

‎Modules/_io/fileio.c‎

Lines changed: 24 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -180,9 +180,9 @@ fileio_new(PyTypeObject *type, PyObject *args, PyObject *kwds)
180180
static int
181181
check_fd(int fd)
182182
{
183-
#if defined(HAVE_FSTAT)
184-
struct stat buf;
185-
if (!_PyVerify_fd(fd) || (fstat(fd, &buf) < 0 && errno == EBADF)) {
183+
#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
184+
struct _Py_stat_struct buf;
185+
if (!_PyVerify_fd(fd) || (_Py_fstat(fd, &buf) < 0 && errno == EBADF)) {
186186
PyObject *exc;
187187
char *msg = strerror(EBADF);
188188
exc = PyObject_CallFunction(PyExc_OSError, "(is)",
@@ -222,8 +222,8 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
222222
#elif !defined(MS_WINDOWS)
223223
int *atomic_flag_works = NULL;
224224
#endif
225-
#ifdef HAVE_FSTAT
226-
struct stat fdfstat;
225+
#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
226+
struct _Py_stat_struct fdfstat;
227227
#endif
228228
int async_err = 0;
229229

@@ -420,9 +420,11 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
420420
}
421421

422422
self->blksize = DEFAULT_BUFFER_SIZE;
423-
#ifdef HAVE_FSTAT
424-
if (fstat(self->fd, &fdfstat) < 0)
423+
#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
424+
if (_Py_fstat(self->fd, &fdfstat) < 0) {
425+
PyErr_SetFromErrno(PyExc_OSError);
425426
goto error;
427+
}
426428
#if defined(S_ISDIR) && defined(EISDIR)
427429
/* On Unix, open will succeed for directories.
428430
In Python, there should be no file objects referring to
@@ -437,7 +439,7 @@ fileio_init(PyObject *oself, PyObject *args, PyObject *kwds)
437439
if (fdfstat.st_blksize > 1)
438440
self->blksize = fdfstat.st_blksize;
439441
#endif /* HAVE_STRUCT_STAT_ST_BLKSIZE */
440-
#endif /* HAVE_FSTAT */
442+
#endif /* HAVE_FSTAT || MS_WINDOWS */
441443

442444
#if defined(MS_WINDOWS) || defined(__CYGWIN__)
443445
/* don't translate newlines (\r\n <=> \n) */
@@ -603,17 +605,7 @@ fileio_readinto(fileio *self, PyObject *args)
603605
return PyLong_FromSsize_t(n);
604606
}
605607

606-
#ifndef HAVE_FSTAT
607-
608-
static PyObject *
609-
fileio_readall(fileio *self)
610-
{
611-
_Py_IDENTIFIER(readall);
612-
return _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type,
613-
&PyId_readall, "O", self);
614-
}
615-
616-
#else
608+
#if defined(HAVE_FSTAT) || defined(MS_WINDOWS)
617609

618610
static size_t
619611
new_buffersize(fileio *self, size_t currentsize)
@@ -637,7 +629,7 @@ new_buffersize(fileio *self, size_t currentsize)
637629
static PyObject *
638630
fileio_readall(fileio *self)
639631
{
640-
struct stat st;
632+
struct _Py_stat_struct st;
641633
Py_off_t pos, end;
642634
PyObject *result;
643635
Py_ssize_t bytes_read = 0;
@@ -655,7 +647,7 @@ fileio_readall(fileio *self)
655647
#else
656648
pos = lseek(self->fd, 0L, SEEK_CUR);
657649
#endif
658-
if (fstat(self->fd, &st) == 0)
650+
if (_Py_fstat(self->fd, &st) == 0)
659651
end = st.st_size;
660652
else
661653
end = (Py_off_t)-1;
@@ -729,7 +721,17 @@ fileio_readall(fileio *self)
729721
return result;
730722
}
731723

732-
#endif /* HAVE_FSTAT */
724+
#else
725+
726+
static PyObject *
727+
fileio_readall(fileio *self)
728+
{
729+
_Py_IDENTIFIER(readall);
730+
return _PyObject_CallMethodId((PyObject*)&PyRawIOBase_Type,
731+
&PyId_readall, "O", self);
732+
}
733+
734+
#endif /* HAVE_FSTAT || MS_WINDOWS */
733735

734736
static PyObject *
735737
fileio_read(fileio *self, PyObject *args)

‎Modules/main.c‎

Lines changed: 2 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -752,9 +752,8 @@ Py_Main(int argc, wchar_t **argv)
752752
}
753753
}
754754
{
755-
/* XXX: does this work on Win/Win64? (see posix_fstat) */
756-
struct stat sb;
757-
if (fstat(fileno(fp), &sb) == 0 &&
755+
struct _Py_stat_struct sb;
756+
if (_Py_fstat(fileno(fp), &sb) == 0 &&
758757
S_ISDIR(sb.st_mode)) {
759758
fprintf(stderr, "%ls: '%ls' is a directory, cannot continue\n", argv[0], filename);
760759
fclose(fp);

‎Modules/mmapmodule.c‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -459,8 +459,8 @@ mmap_size_method(mmap_object *self,
459459

460460
#ifdef UNIX
461461
{
462-
struct stat buf;
463-
if (-1 == fstat(self->fd, &buf)) {
462+
struct _Py_stat_struct buf;
463+
if (-1 == _Py_fstat(self->fd, &buf)) {
464464
PyErr_SetFromErrno(PyExc_OSError);
465465
return NULL;
466466
}
@@ -1107,7 +1107,7 @@ static PyObject *
11071107
new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
11081108
{
11091109
#ifdef HAVE_FSTAT
1110-
struct stat st;
1110+
struct _Py_stat_struct st;
11111111
#endif
11121112
mmap_object *m_obj;
11131113
PyObject *map_size_obj = NULL;
@@ -1174,7 +1174,7 @@ new_mmap_object(PyTypeObject *type, PyObject *args, PyObject *kwdict)
11741174
(void)fcntl(fd, F_FULLFSYNC);
11751175
#endif
11761176
#ifdef HAVE_FSTAT
1177-
if (fd != -1 && fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
1177+
if (fd != -1 && _Py_fstat(fd, &st) == 0 && S_ISREG(st.st_mode)) {
11781178
if (map_size == 0) {
11791179
if (st.st_size == 0) {
11801180
PyErr_SetString(PyExc_ValueError,

0 commit comments

Comments
 (0)