SCIP indexer

.cargo/config.toml

@@ -13,6 +13,7 @@ POLARS_TEST_PATH = { value = "pyrefly/lib/test/polars/third-party", relative = t
 NARWHALS_TEST_PATH = { value = "pyrefly/lib/test/narwhals/third-party", relative = true }
 SQLALCHEMY_TEST_PATH = { value = "pyrefly/lib/test/sqlalchemy/third-party", relative = true }
 GLEAN_SNAPSHOTS_PATH = { value = "pyrefly/lib/report/glean/snapshots", relative = true }
+SCIP_SNAPSHOTS_PATH = { value = "pyrefly/lib/report/scip/snapshots", relative = true }
 COVERAGE_TEST_PATH = { value = "pyrefly/lib/test/coverage/test_files", relative = true }
 STUBGEN_TEST_PATH = { value = "pyrefly/lib/test/stubgen", relative = true }
 SHAPE_EXTENSIONS_TEST_PATH = { value = "tensor-shapes/pyrefly-shape-extensions", relative = true }

.gitattributes

@@ -1 +1,2 @@
 *.mdx text eol=lf
+pyrefly/lib/report/scip/snapshots/** text eol=lf

crates/pyrefly_python/src/ast.rs

@@ -46,6 +46,7 @@ use ruff_python_ast::visitor::Visitor;
 use ruff_python_ast::visitor::source_order::SourceOrderVisitor;
 use ruff_python_ast::visitor::source_order::TraversalSignal;
 use ruff_python_ast::visitor::source_order::walk_node;
+use ruff_python_ast::visitor::source_order::walk_stmt;
 use ruff_python_parser::ParseError;
 use ruff_python_parser::ParseOptions;
 use ruff_python_parser::Parsed;
@@ -471,6 +472,32 @@ impl Ast {
         )
     }
 
+    /// Like `module.visit`, but only calls `f` on expressions that contain `position`, and
+    /// skips statements that don't contain it.
+    pub fn visit_covering_exprs<'a>(
+        module: &'a ModModule,
+        position: TextSize,
+        f: &mut dyn FnMut(&'a Expr),
+    ) {
+        struct X<'a, 'f> {
+            position: TextSize,
+            f: &'f mut dyn FnMut(&'a Expr),
+        }
+        impl<'a> SourceOrderVisitor<'a> for X<'a, '_> {
+            fn visit_stmt(&mut self, x: &'a Stmt) {
+                if x.range().contains_inclusive(self.position) {
+                    walk_stmt(self, x);
+                }
+            }
+            fn visit_expr(&mut self, x: &'a Expr) {
+                if x.range().contains_inclusive(self.position) {
+                    (self.f)(x);
+                }
+            }
+        }
+        X { position, f }.visit_body(&module.body);
+    }
+
     /// Given a module and a position, find all AST nodes that "cover" the position.
     /// Return a vector of AST nodes sorted by the node's range, where the "innermost" node that covers the
     /// position comes first, and parent nodes of that covering node come later.

pyrefly/Cargo.toml

@@ -66,6 +66,7 @@ parking_lot = { version = "0.12.1", features = ["arc_lock", "send_guard"] }
 parse-display = "0.8.2"
 paste = "1.0.14"
 percent-encoding = "2.1"
+protobuf = "3.7.2"
 pyrefly_build = { path = "../crates/pyrefly_build" }
 pyrefly_bundled = { path = "../crates/pyrefly_bundled", default-features = false }
 pyrefly_config = { path = "../crates/pyrefly_config" }
@@ -83,6 +84,7 @@ ruff_python_ast = { version = "0.0.14", features = ["serde"] }
 ruff_python_parser = "0.0.14"
 ruff_source_file = "0.0.14"
 ruff_text_size = "0.0.14"
+scip = "0.10.0"
 serde = { version = "1.0.229", features = ["derive", "rc"] }
 serde_json = { version = "1.0.154", features = ["alloc", "float_roundtrip", "raw_value", "unbounded_depth"] }
 serde_repr = "0.1.21"

pyrefly/lib/binding/class.rs

@@ -261,7 +261,7 @@ impl<'a> BindingsBuilder<'a> {
         let mut pydantic_config_dict = PydanticConfigDict::default();
         let docstring_range = Docstring::range_from_stmts(x.body.as_slice());
         let body = mem::take(&mut x.body);
-        let field_docstrings = self.extract_field_docstrings(&body);
+        let field_docstrings = Self::extract_field_docstrings(&body);
         let pydantic_before_validator_fields = self.extract_field_validator_fields(&body);
         let mut attrs_decorators = AttrsDecoratorMethods::default();
         collect_attrs_decorator_methods(&body, &mut attrs_decorators);
@@ -647,8 +647,7 @@ impl<'a> BindingsBuilder<'a> {
     }
 
     /// Extracts docstrings for each field, mapping the field's range to the docstring's range.
-    fn extract_field_docstrings(
-        &self,
+    pub fn extract_field_docstrings(
         body: &[ruff_python_ast::Stmt],
     ) -> SmallMap<TextRange, TextRange> {
         use ruff_python_ast::Expr;

pyrefly/lib/commands/check.rs

@@ -334,6 +334,15 @@ struct OutputArgs {
     /// stdlib then record those source files instead of stub paths. Affects Glean output only.
     #[arg(long, value_name = "DIR", requires = "report_glean")]
     glean_stdlib_root: Option<String>,
+    /// Generate a SCIP index for the project
+    #[arg(long, value_name = "OUTPUT_FILE")]
+    report_scip: Option<PathBuf>,
+    /// Package name for the project's symbols in the SCIP index. Defaults to the name in `pyproject.toml`.
+    #[arg(long, requires = "report_scip")]
+    report_scip_project_name: Option<String>,
+    /// Package version for the project's symbols in the SCIP index. Defaults to the version in `pyproject.toml`.
+    #[arg(long, requires = "report_scip")]
+    report_scip_project_version: Option<String>,
     /// Generate a Pysa-compatible JSON file for each module. Pysa treats every type error
     /// as a failure, so this defaults `--min-severity` to `error`.
     #[arg(long, value_name = "OUTPUT_FILE")]
@@ -506,6 +515,9 @@ impl OutputArgs {
             report_glean_ownership,
             glean_module_names,
             glean_stdlib_root,
+            report_scip,
+            report_scip_project_name,
+            report_scip_project_version,
             report_pysa,
             report_pysa_format,
             report_demand_tree,
@@ -552,6 +564,9 @@ impl OutputArgs {
             || *report_glean_ownership
             || glean_module_names.is_some()
             || glean_stdlib_root.is_some()
+            || report_scip.is_some()
+            || report_scip_project_name.is_some()
+            || report_scip_project_version.is_some()
             || report_pysa.is_some()
             || !matches!(report_pysa_format, report::pysa::PysaFormat::Capnp)
             || report_demand_tree.is_some()
@@ -1601,7 +1616,7 @@ impl IncrementalCheckCommand {
             )
         });
         let defaults = args.output.resolve(config.as_deref());
-        let run = args.prepare_cli_run(timings, transaction, handles, &defaults)?;
+        let run = args.prepare_cli_run(timings, transaction, version, handles, &defaults)?;
         check.run(|transaction, handles, sourcedb_errors| {
             let result = args.finish_cli_run(
                 run,
@@ -1648,6 +1663,7 @@ struct PreparedCliRun {
     memory_trace: MemoryUsageTrace,
     demand_tree_subscriber: Option<TestSubscriber>,
     type_check_start: Instant,
+    scip: Option<Arc<report::scip::ScipWriter<BufWriter<File>>>>,
 }
 
 impl CheckArgs {
@@ -1802,7 +1818,8 @@ impl CheckArgs {
             || self.output.report_trace.is_some()
             || self.output.report_glean.is_some();
         RequireLevels {
-            specified: if retain {
+            // SCIP solves the dependencies it refers to once the indexed files are written.
+            specified: if retain || self.output.report_scip.is_some() {
                 Require::Everything
             } else {
                 Require::Errors
@@ -1831,7 +1848,7 @@ impl CheckArgs {
         require: Require,
         upsell: UpsellDecision,
     ) -> anyhow::Result<(CommandExitStatus, Vec<Error>)> {
-        let run = self.prepare_cli_run(timings, transaction, handles, defaults)?;
+        let run = self.prepare_cli_run(timings, transaction, version, handles, defaults)?;
         transaction.run(handles, require, None);
         self.finish_cli_run(
             run,
@@ -1848,6 +1865,7 @@ impl CheckArgs {
         &self,
         timings: Timings,
         transaction: &mut Transaction,
+        version: &str,
         handles: &[Handle],
         defaults: &OutputDefaults,
     ) -> anyhow::Result<PreparedCliRun> {
@@ -1908,6 +1926,41 @@ impl CheckArgs {
             transaction.set_cinderx_reporter(Some(cinderx_reporter));
         }
 
+        // Each indexed file is written as soon as it is solved, while its answers are still in
+        // memory, so they don't have to be kept for the whole run.
+        let scip = match &self.output.report_scip {
+            Some(path) => {
+                let relative_to = self.output.relative_to.as_ref().map_or_else(
+                    || std::env::current_dir().ok().unwrap_or_default(),
+                    |x| PathBuf::from_str(x.as_str()).unwrap(),
+                );
+                let project = report::scip::Package::project(
+                    &relative_to,
+                    self.output.report_scip_project_name.clone(),
+                    self.output.report_scip_project_version.clone(),
+                )?;
+                let scip = Arc::new(report::scip::ScipWriter::new(
+                    BufWriter::new(File::create(path)?),
+                    handles,
+                    &relative_to,
+                    project,
+                    version,
+                )?);
+                let hook_scip = scip.dupe();
+                transaction.set_solutions_hook(Some(Box::new(move |handle, transaction| {
+                    if let Err(e) = hook_scip.write_document(transaction, handle) {
+                        error!(
+                            "while writing SCIP document for `{}`: {e:#}",
+                            handle.module()
+                        );
+                    }
+                })));
+                transaction.set_evict_after_solutions_hook(true);
+                Some(scip)
+            }
+            None => None,
+        };
+
         let type_check_start = Instant::now();
         let demand_tree_subscriber = if self.output.report_demand_tree.is_some() {
             transaction.set_demand_collector(Some(DemandCollector::new()));
@@ -1924,6 +1977,7 @@ impl CheckArgs {
             memory_trace,
             demand_tree_subscriber,
             type_check_start,
+            scip,
         })
     }
 
@@ -1942,6 +1996,7 @@ impl CheckArgs {
             mut memory_trace,
             demand_tree_subscriber,
             type_check_start,
+            scip,
         } = run;
         transaction.set_subscriber(None);
 
@@ -2351,6 +2406,16 @@ impl CheckArgs {
                 )?;
             }
         }
+        if let Some(scip) = scip
+            && let Some(path) = &self.output.report_scip
+        {
+            transaction.set_solutions_hook(None);
+            transaction.set_evict_after_solutions_hook(false);
+            let scip = Arc::into_inner(scip).expect("the solutions hook holds the only other copy");
+            scip.finish(transaction)
+                .and_then(|mut writer| Ok(writer.flush()?))
+                .with_context(|| format!("while writing SCIP index to `{}`", path.display()))?;
+        }
         if let Some(pysa_reporter) = transaction.take_pysa_reporter() {
             report::pysa::write_project_file(&pysa_reporter, transaction, handles, &output_errors)?;
         }
@@ -2403,6 +2468,7 @@ mod tests {
     use std::path::PathBuf;
     use std::sync::Arc;
 
+    use protobuf::Message as _;
     use pyrefly_config::config::ConfigScope;
     use pyrefly_python::module::Module;
     use pyrefly_python::module_name::ModuleName;
@@ -2659,6 +2725,57 @@ def go(w: Widget) -> int:
         assert!(!facts.contains("main.Widget"));
     }
 
+    #[test]
+    fn check_writes_scip_report() {
+        let temp = TempDir::new().unwrap();
+        let root = temp.path().join("project");
+        fs::create_dir(&root).unwrap();
+        let module = root.join("main.py");
+        fs::write(&module, "x = 1\n").unwrap();
+        let scip_path = temp.path().join("index.scip");
+
+        let args = CheckArgs::parse_from([
+            "check",
+            "--report-scip",
+            scip_path.to_str().unwrap(),
+            "--report-scip-project-name",
+            "demo",
+            "--relative-to",
+            root.to_str().unwrap(),
+        ]);
+        let (status, errors, _) = args
+            .run_once(
+                "test",
+                Box::new(TestIncludes {
+                    root: root.clone(),
+                    initial_files: vec![module],
+                }),
+                test_config_finder(&root),
+                UpsellDecision::Skip,
+                ThreadCount::Inline,
+            )
+            .unwrap();
+        assert!(errors.is_empty(), "unexpected type errors: {errors:#?}");
+        assert!(matches!(status, CommandExitStatus::Success));
+
+        let index = scip::types::Index::parse_from_bytes(&fs::read(&scip_path).unwrap()).unwrap();
+        assert_eq!(index.documents.len(), 1);
+        assert_eq!(index.documents[0].relative_path, "main.py");
+        assert_eq!(
+            index.documents[0].occurrences[0].symbol,
+            "scip-python python demo . main/__init__:"
+        );
+        let metadata = index.metadata.unwrap();
+        assert_eq!(metadata.tool_info.name, "scip-pyrefly");
+        assert_eq!(metadata.tool_info.version, "test");
+        assert!(
+            metadata.project_root.starts_with("file://")
+                && metadata.project_root.ends_with("/project/"),
+            "unexpected project root: {}",
+            metadata.project_root
+        );
+    }
+
     #[test]
     fn incremental_check_commits_after_glean_loads_source_dependency() {
         let temp = TempDir::new().unwrap();

pyrefly/lib/lsp/non_wasm/server.rs

@@ -5981,6 +5981,7 @@ impl Server {
             HoverOptions {
                 show_go_to_links,
                 verbosity_level,
+                field_docstring_at_definition: false,
             },
         ))
     }