[3.14] Add MSan to CI (GH-158625) (#158834)

* Run a job that the test suite with MSan to the CI (#158625)

* Run the test suite with MSan in CI

* Additional fixes

* Add `_Py_MSAN_UNPOISON_STRING`

* Apply Victor's suggestions

Co-authored-by: Victor Stinner <victor.stinner@gmail.com>

* Apply Victor's suggestions

Co-authored-by: Victor Stinner <victor.stinner@gmail.com>

---------

Co-authored-by: Victor Stinner <victor.stinner@gmail.com>
(cherry picked from commit b93fb19)

* Mark bytes returned by getrandom as initialized for MSan

* Handle invalid non-ASCII struct formats in the fuzz harness

---------

Co-authored-by: Stan Ulbrych <stan@python.org>
Co-authored-by: Victor Stinner <victor.stinner@gmail.com>

Author: 3 people

.github/workflows/build.yml

@@ -589,6 +589,9 @@ jobs:
         - check-name: Undefined behavior
           sanitizer: UBSan
           free-threading: false
+        - check-name: Memory
+          sanitizer: MSan
+          free-threading: false
     uses: ./.github/workflows/reusable-san.yml
     with:
       sanitizer: ${{ matrix.sanitizer }}

.github/workflows/reusable-san.yml

@@ -67,13 +67,26 @@ jobs:
             || ''
           }}
     - name: UBSan option setup
-      if: inputs.sanitizer != 'TSan'
+      if: inputs.sanitizer == 'UBSan'
       run: >-
         echo
         "UBSAN_OPTIONS=${SAN_LOG_OPTION}"
         >> "$GITHUB_ENV"
       env:
         SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
+    - name: MSan option setup
+      if: inputs.sanitizer == 'MSan'
+      run: |
+        echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV"
+        # MSan reports false positives for memory initialized by libraries
+        # that are not built with MSan, so disable modules that use them.
+        {
+          echo '*disabled*'
+          echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib'
+          echo '_lzma _sqlite3 _ssl _tkinter _uuid _zstd readline zlib'
+        } > Modules/Setup.local
+      env:
+        SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
@@ -98,6 +111,8 @@ jobs:
     # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21
     # compile-time blowup on some interpreter files.
     # (https://github.com/llvm/llvm-project/issues/179695)
+    # MSan uses --with-assertions instead of --with-pydebug because its
+    # hooks on the Python memory allocators hide uninitialized reads.
     - name: Configure CPython
       run: >-
         ./configure
@@ -106,9 +121,11 @@ jobs:
         ${{
           inputs.sanitizer == 'TSan'
           && '--with-thread-sanitizer'
+          || inputs.sanitizer == 'MSan'
+          && '--with-memory-sanitizer'
           || '--with-undefined-behavior-sanitizer'
         }}
-        --with-pydebug
+        ${{ inputs.sanitizer == 'MSan' && '--with-assertions' || '--with-pydebug' }}
         ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }}
         ${{ inputs.free-threading && '--disable-gil' || '' }}
     - name: Build CPython

Doc/using/configure.rst

@@ -920,6 +920,10 @@ Debug options
 
    Enable MemorySanitizer allocation error detector, ``msan`` (default is no).
 
+   MSan reports false positives for memory initialized by libraries that are
+   not built with MSan, so either build all dependencies with MSan or disable
+   the extension modules that use them in :file:`Modules/Setup.local`.
+
    .. versionadded:: 3.6
 
 .. option:: --with-undefined-behavior-sanitizer

Include/pyport.h

@@ -577,6 +577,8 @@ extern "C" {
 #    if !defined(_Py_MEMORY_SANITIZER)
 #      define _Py_MEMORY_SANITIZER
 #      define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
+#      define _Py_MSAN_UNPOISON(PTR, SIZE)  (__msan_unpoison(PTR, SIZE))
+#      define _Py_MSAN_UNPOISON_STRING(STR)  (__msan_unpoison_string(STR))
 #    endif
 #  endif
 #  if __has_feature(address_sanitizer)
@@ -615,6 +617,12 @@ extern "C" {
 #ifndef _Py_NO_SANITIZE_MEMORY
 #  define _Py_NO_SANITIZE_MEMORY
 #endif
+#ifndef _Py_MSAN_UNPOISON
+#  define _Py_MSAN_UNPOISON(PTR, SIZE)
+#endif
+#ifndef _Py_MSAN_UNPOISON_STRING
+#  define _Py_MSAN_UNPOISON_STRING(STR)
+#endif
 
 /* AIX has __bool__ redefined in it's system header file. */
 #if defined(_AIX) && defined(__bool__)

Lib/test/test_faulthandler.py

@@ -34,8 +34,8 @@
 
 
 def skip_if_sanitizer_signal(signame):
-    return support.skip_if_sanitizer(f"TSAN/UBSan itercepts {signame}",
-                                     thread=True, ub=True)
+    return support.skip_if_sanitizer(f"TSan/UBSan/MSan intercepts {signame}",
+                                     thread=True, ub=True, memory=True)
 
 
 def expected_traceback(lineno1, lineno2, header, min_count=1):

Lib/test/test_xxtestfuzz.py

@@ -18,6 +18,7 @@ def test_sample_input_smoke_test(self):
         _xxtestfuzz.run(b"1")
         _xxtestfuzz.run(b"AAAAAAA")
         _xxtestfuzz.run(b"AAAAAA\0")
+        _xxtestfuzz.run(b"\xff\0")
 
 
 if __name__ == "__main__":

Modules/_testinternalcapi.c

@@ -1489,8 +1489,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self,
 static PyObject *
 check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
 {
-    /* ASan or TSan would report an use-after-free error */
-#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER)
+    /* ASan, MSan or TSan would report an error. */
+#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) || defined(_Py_MEMORY_SANITIZER)
     Py_RETURN_NONE;
 #else
     PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type);

Modules/_xxtestfuzz/fuzzer.c

@@ -133,6 +133,10 @@ static int fuzz_struct_unpack(const char* data, size_t size) {
     if (unpacked == NULL && PyErr_ExceptionMatches(PyExc_SystemError)) {
         PyErr_Clear();
     }
+    /* Ignore any ValueError, these are triggered by non-ASCII format. */
+    if (unpacked == NULL && PyErr_ExceptionMatches(PyExc_ValueError)) {
+        PyErr_Clear();
+    }
     /* Ignore any struct.error exceptions, these can be caused by invalid
        formats or incomplete buffers both of which are common. */
     if (unpacked == NULL && PyErr_ExceptionMatches(struct_error)) {

Modules/posixmodule.c

@@ -9660,6 +9660,7 @@ os_getlogin_impl(PyObject *module)
         errno = old_errno;
     }
     else {
+        _Py_MSAN_UNPOISON(name, sizeof(name));
         result = PyUnicode_DecodeFSDefault(name);
     }
 #else
@@ -16823,6 +16824,8 @@ os_getrandom_impl(PyObject *module, Py_ssize_t size, int flags)
         goto error;
     }
 
+    _Py_MSAN_UNPOISON(PyBytes_AS_STRING(bytes), n);
+
     if (n != size) {
         _PyBytes_Resize(&bytes, n);
     }

Modules/socketmodule.c

@@ -752,7 +752,9 @@ set_herror(socket_state *state, int h_error)
     PyObject *v;
 
 #ifdef HAVE_HSTRERROR
-    v = Py_BuildValue("(iN)", h_error, decode_error_message(hstrerror(h_error)));
+    const char *errmsg = hstrerror(h_error);
+    _Py_MSAN_UNPOISON_STRING(errmsg);
+    v = Py_BuildValue("(iN)", h_error, decode_error_message(errmsg));
 #else
     v = Py_BuildValue("(is)", h_error, "host not found");
 #endif
@@ -779,7 +781,9 @@ set_gaierror(socket_state *state, int error)
 #endif
 
 #ifdef HAVE_GAI_STRERROR
-    v = Py_BuildValue("(iN)", error, decode_error_message(gai_strerror(error)));
+    const char *errmsg = gai_strerror(error);
+    _Py_MSAN_UNPOISON_STRING(errmsg);
+    v = Py_BuildValue("(iN)", error, decode_error_message(errmsg));
 #else
     v = Py_BuildValue("(is)", error, "getaddrinfo failed");
 #endif
@@ -6408,6 +6412,7 @@ socket_getservbyport(PyObject *self, PyObject *args)
         PyErr_SetString(PyExc_OSError, "port/proto not found");
         return NULL;
     }
+    _Py_MSAN_UNPOISON_STRING(sp->s_name);
     return PyUnicode_FromString(sp->s_name);
 }