py/mpconfig.h: Be stricter when autodetecting machine endianness.

dpgeorge · dpgeorge · commit 9630376dbca0 · 2018-05-11T21:51:34.000+10:00
This patch changes 2 things in the endianness detection: 1. Don't assume that __BYTE_ORDER__ not being __ORDER_LITTLE_ENDIAN__ means that the machine is big endian, so add an explicit check that this macro is indeed __ORDER_BIG_ENDIAN__ (same with __BYTE_ORDER, __LITTLE_ENDIAN and __BIG_ENDIAN). A machine could have PDP endianness. 2. Remove the checks which base their autodetection decision on whether any little or big endian macros are defined (eg __LITTLE_ENDIAN__ or __BIG_ENDIAN__). Just because a system defines these does not mean it has that endianness. See issue adafruit#3760.
diff --git a/py/mpconfig.h b/py/mpconfig.h
@@ -1256,29 +1256,26 @@ typedef double mp_float_t;
 #elif defined(MP_ENDIANNESS_BIG)
 #define MP_ENDIANNESS_LITTLE (!MP_ENDIANNESS_BIG)
 #else
-  // Endiannes not defined by port so try to autodetect it.
+  // Endianness not defined by port so try to autodetect it.
   #if defined(__BYTE_ORDER__)
     #if __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__
       #define MP_ENDIANNESS_LITTLE (1)
-    #else
+    #elif __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__
       #define MP_ENDIANNESS_LITTLE (0)
     #endif
-  #elif defined(__LITTLE_ENDIAN__) || defined(__LITTLE_ENDIAN) || defined (_LITTLE_ENDIAN)
-    #define MP_ENDIANNESS_LITTLE (1)
-  #elif defined(__BIG_ENDIAN__) || defined(__BIG_ENDIAN) || defined (_BIG_ENDIAN)
-    #define MP_ENDIANNESS_LITTLE (0)
   #else
     #include <endian.h>
       #if defined(__BYTE_ORDER)
         #if __BYTE_ORDER == __LITTLE_ENDIAN
           #define MP_ENDIANNESS_LITTLE (1)
-        #else
+        #elif __BYTE_ORDER == __BIG_ENDIAN
           #define MP_ENDIANNESS_LITTLE (0)
         #endif
-      #else
-        #error endianness not defined and cannot detect it
       #endif
   #endif
+  #ifndef MP_ENDIANNESS_LITTLE
+    #error endianness not defined and cannot detect it
+  #endif
   #define MP_ENDIANNESS_BIG (!MP_ENDIANNESS_LITTLE)
 #endif
 
