Issue #13863: fix incorrect .pyc timestamps on Windows / NTFS (apparently due to buggy fstat)

mdickinson · mdickinson · commit 344a5a48014b · 2012-12-24T16:33:18.000Z
diff --git a/Lib/test/test_import.py b/Lib/test/test_import.py
@@ -5,6 +5,7 @@
 import py_compile
 import random
 import stat
+import struct
 import sys
 import unittest
 import textwrap
@@ -350,6 +351,46 @@ def test_timestamp_overflow(self):
             del sys.path[0]
             remove_files(TESTFN)
 
+    def test_pyc_mtime(self):
+        # Test for issue #13863: .pyc timestamp sometimes incorrect on Windows.
+        sys.path.insert(0, os.curdir)
+        try:
+            # Jan 1, 2012; Jul 1, 2012.
+            mtimes = 1325376000, 1341100800
+
+            # Different names to avoid running into import caching.
+            tails = "spam", "eggs"
+            for mtime, tail in zip(mtimes, tails):
+                module = TESTFN + tail
+                source = module + ".py"
+                compiled = source + ('c' if __debug__ else 'o')
+
+                # Create a new Python file with the given mtime.
+                with open(source, 'w') as f:
+                    f.write("# Just testing\nx=1, 2, 3\n")
+                os.utime(source, (mtime, mtime))
+
+                # Generate the .pyc/o file; if it couldn't be created
+                # for some reason, skip the test.
+                m = __import__(module)
+                if not os.path.exists(compiled):
+                    unlink(source)
+                    self.skipTest("Couldn't create .pyc/.pyo file.")
+
+                # Actual modification time of .py file.
+                mtime1 = int(os.stat(source).st_mtime) & 0xffffffff
+
+                # mtime that was encoded in the .pyc file.
+                with open(compiled, 'rb') as f:
+                    mtime2 = struct.unpack('<L', f.read(8)[4:])[0]
+
+                unlink(compiled)
+                unlink(source)
+
+                self.assertEqual(mtime1, mtime2)
+        finally:
+            sys.path.pop(0)
+
 
 class PycRewritingTests(unittest.TestCase):
     # Test that the `co_filename` attribute on code objects always points
diff --git a/Misc/NEWS b/Misc/NEWS
@@ -9,6 +9,10 @@ What's New in Python 2.7.4
 Core and Builtins
 -----------------
 
+- Issue #13863: Work around buggy 'fstat' implementation on Windows / NTFS that
+  lead to incorrect timestamps (off by one hour) being stored in .pyc files on
+  some systems.
+
 - Issue #16602: When a weakref's target was part of a long deallocation
   chain, the object could remain reachable through its weakref even though
   its refcount had dropped to zero.
diff --git a/Python/import.c b/Python/import.c
@@ -904,10 +904,9 @@ open_exclusive(char *filename, mode_t mode)
    remove the file. */
 
 static void
-write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat)
+write_compiled_module(PyCodeObject *co, char *cpathname, struct stat *srcstat, time_t mtime)
 {
     FILE *fp;
-    time_t mtime = srcstat->st_mtime;
 #ifdef MS_WINDOWS   /* since Windows uses different permissions  */
     mode_t mode = srcstat->st_mode & ~S_IEXEC;
     /* Issue #6074: We ensure user write access, so we can delete it later
@@ -993,6 +992,38 @@ update_compiled_module(PyCodeObject *co, char *pathname)
     return 1;
 }
 
+#ifdef MS_WINDOWS
+
+/* Seconds between 1.1.1601 and 1.1.1970 */
+static __int64 secs_between_epochs = 11644473600;
+
+/* Get mtime from file pointer. */
+
+static time_t
+win32_mtime(FILE *fp, char *pathname)
+{
+    __int64 filetime;
+    HANDLE fh;
+    BY_HANDLE_FILE_INFORMATION file_information;
+
+    fh = (HANDLE)_get_osfhandle(fileno(fp));
+    if (fh == INVALID_HANDLE_VALUE ||
+        !GetFileInformationByHandle(fh, &file_information)) {
+        PyErr_Format(PyExc_RuntimeError,
+                     "unable to get file status from '%s'",
+                     pathname);
+        return -1;
+    }
+    /* filetime represents the number of 100ns intervals since
+       1.1.1601 (UTC).  Convert to seconds since 1.1.1970 (UTC). */
+    filetime = (__int64)file_information.ftLastWriteTime.dwHighDateTime << 32 |
+               file_information.ftLastWriteTime.dwLowDateTime;
+    return filetime / 10000000 - secs_between_epochs;
+}
+
+#endif  /* #ifdef MS_WINDOWS */
+
+
 /* Load a source module from a given file and return its module
    object WITH INCREMENTED REFERENCE COUNT.  If there's a matching
    byte-compiled file, use that instead. */
@@ -1006,20 +1037,29 @@ load_source_module(char *name, char *pathname, FILE *fp)
     char *cpathname;
     PyCodeObject *co = NULL;
     PyObject *m;
+    time_t mtime;
 
     if (fstat(fileno(fp), &st) != 0) {
         PyErr_Format(PyExc_RuntimeError,
                      "unable to get file status from '%s'",
                      pathname);
         return NULL;
     }
-    if (sizeof st.st_mtime > 4) {
+
+#ifdef MS_WINDOWS
+    mtime = win32_mtime(fp, pathname);
+    if (mtime == (time_t)-1 && PyErr_Occurred())
+        return NULL;
+#else
+    mtime = st.st_mtime;
+#endif
+    if (sizeof mtime > 4) {
         /* Python's .pyc timestamp handling presumes that the timestamp fits
            in 4 bytes. Since the code only does an equality comparison,
            ordering is not important and we can safely ignore the higher bits
            (collisions are extremely unlikely).
          */
-        st.st_mtime &= 0xFFFFFFFF;
+        mtime &= 0xFFFFFFFF;
     }
     buf = PyMem_MALLOC(MAXPATHLEN+1);
     if (buf == NULL) {
@@ -1028,7 +1068,7 @@ load_source_module(char *name, char *pathname, FILE *fp)
     cpathname = make_compiled_pathname(pathname, buf,
                                        (size_t)MAXPATHLEN + 1);
     if (cpathname != NULL &&
-        (fpc = check_compiled_module(pathname, st.st_mtime, cpathname))) {
+        (fpc = check_compiled_module(pathname, mtime, cpathname))) {
         co = read_compiled_module(cpathname, fpc);
         fclose(fpc);
         if (co == NULL)
@@ -1053,7 +1093,7 @@ load_source_module(char *name, char *pathname, FILE *fp)
             if (b < 0)
                 goto error_exit;
             if (!b)
-                write_compiled_module(co, cpathname, &st);
+                write_compiled_module(co, cpathname, &st, mtime);
         }
     }
     m = PyImport_ExecCodeModuleEx(name, (PyObject *)co, pathname);
