windows/msvc: Nicer handling of asserts and 'invalid' parameters

stinos · dpgeorge · commit 1aa4599ddd7a · 2015-12-26T22:14:16.000Z
The default bahaviour for debug builds is to show dialog boxes for asserts
and invalid parameter handling. This is not so nice in general and causes
the Appveyor debug builds to hang because the io\file_seek.py test passes
a closed file descriptor to lseek. Disable this behaviour by printing
assert messages to the output instead of showing the dialog, and by
disabling 'invalid' parameter handling which causes the affected functions
to just return an error and set errno appropriately.
diff --git a/windows/init.c b/windows/init.c
@@ -27,11 +27,31 @@
 #include <stdlib.h>
 #include <stdio.h>
 #include <windows.h>
+#ifdef _MSC_VER
+#include <crtdbg.h>
+#endif
 #include "sleep.h"
 
 extern BOOL WINAPI console_sighandler(DWORD evt);
 
+#ifdef _MSC_VER
+void invalid_param_handler(const wchar_t *expr, const wchar_t *fun, const wchar_t *file, unsigned int line, uintptr_t p) {
+}
+#endif
+
 void init() {
+#ifdef _MSC_VER
+    // Disable the 'Debug Error!' dialog for assertions failures and the likes,
+    // instead write messages to the debugger output and terminate.
+    _CrtSetReportMode(_CRT_ASSERT, _CRTDBG_MODE_DEBUG);
+    _CrtSetReportMode(_CRT_WARN, _CRTDBG_MODE_DEBUG);
+    _CrtSetReportMode(_CRT_ERROR, _CRTDBG_MODE_DEBUG);
+
+    // Disable 'invalid parameter handling' which is for instance invoked when
+    // passing invalid file descriptors to functions like lseek() and make the
+    // functions called behave properly by setting errno to EBADF/EINVAL/..
+    _set_invalid_parameter_handler(invalid_param_handler);
+#endif
     SetConsoleCtrlHandler(console_sighandler, TRUE);
     init_sleep();
 #ifdef __MINGW32__
