Microoptimise read buffering, metastore abspath, path joining (#20810)

Removing buffering on file reads can make those reads like 20% faster.
This is probably worth a few percent on a warm cache profile I was
looking at.

Removing abspath in metastore also looks like a percent or two on a warm
cache profile.

I don't think using the faster path join from #17949 will really help
the profiles I was looking at, but if we're microoptimising some of
these code paths we might as well. (On a cold profile I do think some of
the variadic os.path.join now add up enough that it could be worth
making a function for them, but whatever)

Author: hauntsaninja

mypy/build.py

@@ -119,6 +119,7 @@
     is_sub_path_normabs,
     is_typeshed_file,
     module_prefix,
+    os_path_join,
     read_py_file,
     time_ref,
     time_spent_us,
@@ -582,7 +583,7 @@ def plugin_error(message: str) -> NoReturn:
             plugin_path, func_name = plugin_path.rsplit(":", 1)
         if plugin_path.endswith(".py"):
             # Plugin paths can be relative to the config file location.
-            plugin_path = os.path.join(os.path.dirname(options.config_file), plugin_path)
+            plugin_path = os_path_join(os.path.dirname(options.config_file), plugin_path)
             if not os.path.isfile(plugin_path):
                 plugin_error(f'Can\'t find plugin "{plugin_path}"')
             # Use an absolute path to avoid populating the cache entry
@@ -1466,7 +1467,7 @@ def _cache_dir_prefix(options: Options) -> str:
         return os.curdir
     cache_dir = options.cache_dir
     pyversion = options.python_version
-    base = os.path.join(cache_dir, "%d.%d" % pyversion)
+    base = os_path_join(cache_dir, "%d.%d" % pyversion)
     return base
 
 
@@ -1475,7 +1476,7 @@ def add_catch_all_gitignore(target_dir: str) -> None:
 
     No-op if the .gitignore already exists.
     """
-    gitignore = os.path.join(target_dir, ".gitignore")
+    gitignore = os_path_join(target_dir, ".gitignore")
     try:
         with open(gitignore, "x") as f:
             print("# Automatically created by mypy", file=f)
@@ -1489,7 +1490,7 @@ def exclude_from_backups(target_dir: str) -> None:
 
     If the CACHEDIR.TAG file exists the function is a no-op.
     """
-    cachedir_tag = os.path.join(target_dir, "CACHEDIR.TAG")
+    cachedir_tag = os_path_join(target_dir, "CACHEDIR.TAG")
     try:
         with open(cachedir_tag, "x") as f:
             f.write("""Signature: 8a477f597d28d172789f06886806bc55
@@ -1539,7 +1540,7 @@ def get_cache_names(id: str, path: str, options: Options) -> tuple[str, str, str
     prefix = os.path.join(*id.split("."))
     is_package = os.path.basename(path).startswith("__init__.py")
     if is_package:
-        prefix = os.path.join(prefix, "__init__")
+        prefix = os_path_join(prefix, "__init__")
 
     deps_json = None
     if options.cache_fine_grained:
@@ -2393,7 +2394,7 @@ def __init__(
             if os.path.isabs(path):
                 self.abspath = path
             else:
-                self.abspath = os.path.normpath(os.path.join(manager.cwd, path))
+                self.abspath = os.path.normpath(os_path_join(manager.cwd, path))
         self.xpath = path or "<string>"
         self.source = source
         self.options = options
@@ -4327,7 +4328,7 @@ def transitive_dep_hash(scc: SCC, graph: Graph) -> bytes:
 
 
 def missing_stubs_file(cache_dir: str) -> str:
-    return os.path.join(cache_dir, "missing_stubs")
+    return os_path_join(cache_dir, "missing_stubs")
 
 
 def record_missing_stub_packages(cache_dir: str, missing_stub_packages: set[str]) -> None:

mypy/metastore.py

@@ -17,6 +17,8 @@
 from collections.abc import Iterable
 from typing import TYPE_CHECKING, Any
 
+from mypy.util import os_path_join
+
 if TYPE_CHECKING:
     # We avoid importing sqlite3 unless we are using it so we can mostly work
     # on semi-broken pythons that are missing it.
@@ -85,24 +87,24 @@ def getmtime(self, name: str) -> float:
         if not self.cache_dir_prefix:
             raise FileNotFoundError()
 
-        return int(os.path.getmtime(os.path.join(self.cache_dir_prefix, name)))
+        return int(os.path.getmtime(os_path_join(self.cache_dir_prefix, name)))
 
     def read(self, name: str) -> bytes:
-        assert os.path.normpath(name) != os.path.abspath(name), "Don't use absolute paths!"
+        assert not os.path.isabs(name), "Don't use absolute paths!"
 
         if not self.cache_dir_prefix:
             raise FileNotFoundError()
 
-        with open(os.path.join(self.cache_dir_prefix, name), "rb") as f:
+        with open(os_path_join(self.cache_dir_prefix, name), "rb", buffering=0) as f:
             return f.read()
 
     def write(self, name: str, data: bytes, mtime: float | None = None) -> bool:
-        assert os.path.normpath(name) != os.path.abspath(name), "Don't use absolute paths!"
+        assert not os.path.isabs(name), "Don't use absolute paths!"
 
         if not self.cache_dir_prefix:
             return False
 
-        path = os.path.join(self.cache_dir_prefix, name)
+        path = os_path_join(self.cache_dir_prefix, name)
         tmp_filename = path + "." + random_string()
         try:
             os.makedirs(os.path.dirname(path), exist_ok=True)
@@ -120,7 +122,7 @@ def remove(self, name: str) -> None:
         if not self.cache_dir_prefix:
             raise FileNotFoundError()
 
-        os.remove(os.path.join(self.cache_dir_prefix, name))
+        os.remove(os_path_join(self.cache_dir_prefix, name))
 
     def commit(self) -> None:
         pass
@@ -132,7 +134,7 @@ def list_all(self) -> Iterable[str]:
         for dir, _, files in os.walk(self.cache_dir_prefix):
             dir = os.path.relpath(dir, self.cache_dir_prefix)
             for file in files:
-                yield os.path.normpath(os.path.join(dir, file))
+                yield os.path.normpath(os_path_join(dir, file))
 
 
 SCHEMA = """
@@ -168,7 +170,7 @@ def __init__(self, cache_dir_prefix: str, sync_off: bool = False) -> None:
             return
 
         os.makedirs(cache_dir_prefix, exist_ok=True)
-        self.db = connect_db(os.path.join(cache_dir_prefix, "cache.db"), sync_off=sync_off)
+        self.db = connect_db(os_path_join(cache_dir_prefix, "cache.db"), sync_off=sync_off)
 
     def _query(self, name: str, field: str) -> Any:
         # Raises FileNotFound for consistency with the file system version