Run a job that the test suite with MSan to the CI (#158625)

* Run the test suite with MSan in CI

* Additional fixes

* Add `_Py_MSAN_UNPOISON_STRING`

* Apply Victor's suggestions

Co-authored-by: Victor Stinner <victor.stinner@gmail.com>

* Apply Victor's suggestions

Co-authored-by: Victor Stinner <victor.stinner@gmail.com>

---------

Co-authored-by: Victor Stinner <victor.stinner@gmail.com>

Author: StanFromIreland

.github/workflows/build.yml

@@ -547,6 +547,9 @@ jobs:
         - check-name: Undefined behavior
           sanitizer: UBSan
           free-threading: false
+        - check-name: Memory
+          sanitizer: MSan
+          free-threading: false
     uses: ./.github/workflows/reusable-san.yml
     with:
       sanitizer: ${{ matrix.sanitizer }}

.github/workflows/reusable-san.yml

@@ -60,7 +60,7 @@ jobs:
             || ''
           }}
     - name: UBSan option setup
-      if: inputs.sanitizer != 'TSan'
+      if: inputs.sanitizer == 'UBSan'
       run: >-
         echo
         "UBSAN_OPTIONS=${SAN_LOG_OPTION}
@@ -69,6 +69,20 @@ jobs:
         >> "$GITHUB_ENV"
       env:
         SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
+    - name: MSan option setup
+      if: inputs.sanitizer == 'MSan'
+      run: |
+        echo "MSAN_OPTIONS=${SAN_LOG_OPTION} allocator_may_return_null=1 handle_segv=0" >> "$GITHUB_ENV"
+        # MSan reports false positives for memory initialized by libraries
+        # that are not built with MSan, so disable modules that use them.
+        # _remote_debugging links to libzstd directly, but we unpoision the memory.
+        {
+          echo '*disabled*'
+          echo '_bz2 _ctypes _curses _curses_panel _dbm _decimal _gdbm _hashlib'
+          echo '_lzma _sqlite3 _ssl _tkinter _uuid _zstd readline zlib'
+        } > Modules/Setup.local
+      env:
+        SAN_LOG_OPTION: log_path=${{ github.workspace }}/san_log
     - name: Add ccache to PATH
       run: |
         echo "PATH=/usr/lib/ccache:$PATH" >> "$GITHUB_ENV"
@@ -93,6 +107,8 @@ jobs:
     # gh-157958: -O2 instead of the pydebug default -Og to avoid a clang 21
     # compile-time blowup on some interpreter files.
     # (https://github.com/llvm/llvm-project/issues/179695)
+    # MSan uses --with-assertions instead of --with-pydebug because its
+    # hooks on the Python memory allocators hide uninitialized reads.
     - name: Configure CPython
       run: >-
         ./configure
@@ -101,9 +117,11 @@ jobs:
         ${{
           inputs.sanitizer == 'TSan'
           && '--with-thread-sanitizer'
+          || inputs.sanitizer == 'MSan'
+          && '--with-memory-sanitizer'
           || '--with-undefined-behavior-sanitizer --with-strict-overflow'
         }}
-        --with-pydebug
+        ${{ inputs.sanitizer == 'MSan' && '--with-assertions' || '--with-pydebug' }}
         ${{ inputs.sanitizer == 'TSan' && '--with-openssl="$OPENSSL_DIR" --with-openssl-rpath=auto' || '' }}
         ${{ inputs.free-threading && '--disable-gil' || '' }}
     - name: Build CPython

Doc/using/configure.rst

@@ -1026,6 +1026,10 @@ Debug options
 
    Enable MemorySanitizer allocation error detector, ``msan`` (default is no).
 
+   MSan reports false positives for memory initialized by libraries that are
+   not built with MSan, so either build all dependencies with MSan or disable
+   the extension modules that use them in :file:`Modules/Setup.local`.
+
    .. versionadded:: 3.6
 
 .. option:: --with-undefined-behavior-sanitizer

Include/pyport.h

@@ -558,6 +558,7 @@ extern "C" {
 #      define _Py_MEMORY_SANITIZER
 #      define _Py_NO_SANITIZE_MEMORY __attribute__((no_sanitize_memory))
 #      define _Py_MSAN_UNPOISON(PTR, SIZE)  (__msan_unpoison(PTR, SIZE))
+#      define _Py_MSAN_UNPOISON_STRING(STR)  (__msan_unpoison_string(STR))
 #    endif
 #  endif
 #  if __has_feature(address_sanitizer)
@@ -599,6 +600,9 @@ extern "C" {
 #ifndef _Py_MSAN_UNPOISON
 #  define _Py_MSAN_UNPOISON(PTR, SIZE)
 #endif
+#ifndef _Py_MSAN_UNPOISON_STRING
+#  define _Py_MSAN_UNPOISON_STRING(STR)
+#endif
 
 /* AIX has __bool__ redefined in it's system header file. */
 #if defined(_AIX) && defined(__bool__)

Lib/test/_test_multiprocessing.py

@@ -3159,6 +3159,7 @@ def test_imap_and_imap_unordered_with_buffersize_type_validation(
         with self.assertRaisesRegex(expected_exception, expected_regex):
             method(str, range(4), buffersize=buffersize)
 
+    @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
     @warnings_helper.ignore_fork_in_thread_deprecation_warnings()
     @support.subTests('method_name', ("imap", "imap_unordered"))
     def test_imap_and_imap_unordered_when_buffer_is_full(self, method_name):
@@ -3194,6 +3195,7 @@ def produce_args():
         p.terminate()
         p.join()
 
+    @unittest.skipUnless(HAS_SHAREDCTYPES, 'needs sharedctypes')
     @warnings_helper.ignore_fork_in_thread_deprecation_warnings()
     @support.subTests('method_name', ("imap", "imap_unordered"))
     def test_imap_and_imap_unordered_with_buffersize_when_buffer_is_full(

Lib/test/test_cext/__init__.py

@@ -192,6 +192,7 @@ def test_build(self):
         self.check_build('_test_cppext_internal')
 
 
+@support.requires_venv_with_pip()
 def setUpModule():
     global VENV_CONTEXT, PYTHON_EXE
     VENV_CONTEXT = support.setup_venv_with_pip_setuptools('env')

Lib/test/test_faulthandler.py

@@ -34,8 +34,8 @@
 
 
 def skip_if_sanitizer_signal(signame):
-    return support.skip_if_sanitizer(f"TSAN/UBSan itercepts {signame}",
-                                     thread=True, ub=True)
+    return support.skip_if_sanitizer(f"TSan/UBSan/MSan intercepts {signame}",
+                                     thread=True, ub=True, memory=True)
 
 
 def expected_traceback(lineno1, lineno2, header, min_count=1):

Modules/_remote_debugging/binary_io_reader.c

@@ -19,6 +19,10 @@
 #include <zstd.h>
 #endif
 
+#ifdef _Py_MEMORY_SANITIZER
+#  include <sanitizer/msan_interface.h>
+#endif
+
 /* ============================================================================
  * CONSTANTS FOR BINARY FORMAT SIZES
  * ============================================================================ */
@@ -315,6 +319,7 @@ reader_decompress_samples(BinaryReader *reader, const uint8_t *data)
             return -1;
         }
 
+        _Py_MSAN_UNPOISON(output.dst, output.pos);
         total_output += output.pos;
     }
 

Modules/_remote_debugging/binary_io_writer.c

@@ -19,6 +19,10 @@
 #include <zstd.h>
 #endif
 
+#ifdef _Py_MEMORY_SANITIZER
+#  include <sanitizer/msan_interface.h>
+#endif
+
 /* ============================================================================
  * CONSTANTS FOR BINARY FORMAT SIZES
  * ============================================================================ */
@@ -235,6 +239,7 @@ writer_flush_buffer(BinaryWriter *writer)
                 return -1;
             }
 
+            _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos);
             if (output.pos > 0) {
                 if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) {
                     return -1;
@@ -1104,6 +1109,7 @@ binary_writer_finalize(BinaryWriter *writer)
                 return -1;
             }
 
+            _Py_MSAN_UNPOISON(writer->zstd.compressed_buffer, output.pos);
             if (output.pos > 0) {
                 if (fwrite_checked_allow_threads(writer->zstd.compressed_buffer, output.pos, writer->fp) < 0) {
                     return -1;

Modules/_testinternalcapi.c

@@ -469,7 +469,7 @@ next_frame_pointer_is_valid(uintptr_t *frame_pointer, uintptr_t *next_fp,
 #endif
 }
 
-static PyObject *
+static PyObject * _Py_NO_SANITIZE_MEMORY
 manual_unwind_from_fp(uintptr_t *frame_pointer)
 {
     uintptr_t stack_min = 0;
@@ -2110,8 +2110,8 @@ check_pyobject_forbidden_bytes_is_freed(PyObject *self,
 static PyObject *
 check_pyobject_freed_is_freed(PyObject *self, PyObject *Py_UNUSED(args))
 {
-    /* ASan or TSan would report an use-after-free error */
-#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER)
+    /* ASan, MSan or TSan would report an error. */
+#if defined(_Py_ADDRESS_SANITIZER) || defined(_Py_THREAD_SANITIZER) || defined(_Py_MEMORY_SANITIZER)
     Py_RETURN_NONE;
 #else
     PyObject *op = PyObject_CallNoArgs((PyObject *)&PyBaseObject_Type);