From c017e7e52db138ee023b8bda288abff7a3375bae Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sat, 17 Oct 2026 19:39:47 -0400 Subject: [PATCH 01/21] add deps and scaffolding --- Cargo.lock | 31 +++++++++++++++++++ pyrefly/Cargo.toml | 2 ++ pyrefly/lib/commands/check.rs | 58 ++++++++++++++++++++++++++++++++++- pyrefly/lib/report.rs | 1 + pyrefly/lib/report/scip.rs | 36 ++++++++++++++++++++++ 5 files changed, 127 insertions(+), 1 deletion(-) create mode 100644 pyrefly/lib/report/scip.rs diff --git a/Cargo.lock b/Cargo.lock index b521e47d8a..3183ff18df 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2328,6 +2328,26 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "protobuf" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d65a1d4ddae7d8b5de68153b48f6aa3bba8cb002b243dbdbc55a5afbc98f99f4" +dependencies = [ + "once_cell", + "protobuf-support", + "thiserror 1.0.69", +] + +[[package]] +name = "protobuf-support" +version = "3.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3e36c2f31e0a47f9280fb347ef5e461ffcd2c52dd520d8e216b52f93b0b0d7d6" +dependencies = [ + "thiserror 1.0.69", +] + [[package]] name = "psm" version = "0.1.29" @@ -2390,6 +2410,7 @@ dependencies = [ "paste", "percent-encoding", "pretty_assertions", + "protobuf", "pyrefly_bench_harness", "pyrefly_build", "pyrefly_bundled", @@ -2409,6 +2430,7 @@ dependencies = [ "ruff_python_parser", "ruff_source_file", "ruff_text_size", + "scip", "serde", "serde_json", "serde_repr", @@ -3036,6 +3058,15 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scip" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d7f651fbd0f98742a47b58623a7bcfd8fad14455912689e4867a1f43ee7e02c6" +dependencies = [ + "protobuf", +] + [[package]] name = "scopeguard" version = "1.2.0" diff --git a/pyrefly/Cargo.toml b/pyrefly/Cargo.toml index 49708f3cae..9381e79ae0 100644 --- a/pyrefly/Cargo.toml +++ b/pyrefly/Cargo.toml @@ -66,6 +66,7 @@ parking_lot = { version = "0.12.1", features = ["arc_lock", "send_guard"] } parse-display = "0.8.2" paste = "1.0.14" percent-encoding = "2.1" +protobuf = "3.7.2" pyrefly_build = { path = "../crates/pyrefly_build" } pyrefly_bundled = { path = "../crates/pyrefly_bundled", default-features = false } pyrefly_config = { path = "../crates/pyrefly_config" } @@ -83,6 +84,7 @@ ruff_python_ast = { version = "0.0.14", features = ["serde"] } ruff_python_parser = "0.0.14" ruff_source_file = "0.0.14" ruff_text_size = "0.0.14" +scip = "0.10.0" serde = { version = "1.0.229", features = ["derive", "rc"] } serde_json = { version = "1.0.151", features = ["alloc", "float_roundtrip", "raw_value", "unbounded_depth"] } serde_repr = "0.1.21" diff --git a/pyrefly/lib/commands/check.rs b/pyrefly/lib/commands/check.rs index c4fee1f504..6117ae956b 100644 --- a/pyrefly/lib/commands/check.rs +++ b/pyrefly/lib/commands/check.rs @@ -35,6 +35,7 @@ use dupe::Dupe as _; use percent_encoding::AsciiSet; use percent_encoding::CONTROLS; use percent_encoding::utf8_percent_encode; +use protobuf::Message as _; use pyrefly_build::handle::Handle; use pyrefly_config::args::ConfigOverrideArgs; use pyrefly_config::base::Preset; @@ -323,6 +324,9 @@ struct OutputArgs { /// Make each module's file the Glean ownership unit of its facts in the Glean report. #[arg(long, requires = "report_glean")] report_glean_ownership: bool, + /// Generate a SCIP index for the project + #[arg(long, value_name = "OUTPUT_FILE")] + report_scip: Option, /// Generate a Pysa-compatible JSON file for each module #[arg(long, value_name = "OUTPUT_FILE")] report_pysa: Option, @@ -492,6 +496,7 @@ impl OutputArgs { report_timings, report_glean, report_glean_ownership, + report_scip, report_pysa, report_pysa_format, report_demand_tree, @@ -536,6 +541,7 @@ impl OutputArgs { || report_timings.is_some() || report_glean.is_some() || *report_glean_ownership + || report_scip.is_some() || report_pysa.is_some() || !matches!(report_pysa_format, report::pysa::PysaFormat::Capnp) || report_demand_tree.is_some() @@ -1763,7 +1769,8 @@ impl CheckArgs { let retain = self.output.report_binding_memory.is_some() || self.output.debug_info.is_some() || self.output.report_trace.is_some() - || self.output.report_glean.is_some(); + || self.output.report_glean.is_some() + || self.output.report_scip.is_some(); RequireLevels { specified: if retain { Require::Everything @@ -2262,6 +2269,12 @@ impl CheckArgs { )?; } } + if let Some(path) = &self.output.report_scip { + fs_anyhow::write( + path, + report::scip::scip(&relative_to, version).write_to_bytes()?, + )?; + } if let Some(pysa_reporter) = transaction.take_pysa_reporter() { report::pysa::write_project_file(&pysa_reporter, transaction, handles, &output_errors)?; } @@ -2548,6 +2561,49 @@ def go(w: Widget) -> int: assert!(fs::metadata(&glean_files[0]).unwrap().len() > 0); } + #[test] + fn check_writes_scip_report() { + let temp = TempDir::new().unwrap(); + let root = temp.path().join("project"); + fs::create_dir(&root).unwrap(); + let module = root.join("main.py"); + fs::write(&module, "x = 1\n").unwrap(); + let scip_path = temp.path().join("index.scip"); + + let args = CheckArgs::parse_from([ + "check", + "--report-scip", + scip_path.to_str().unwrap(), + "--relative-to", + root.to_str().unwrap(), + ]); + let (status, errors, _) = args + .run_once( + "test", + Box::new(TestIncludes { + root: root.clone(), + initial_files: vec![module], + }), + test_config_finder(&root), + UpsellDecision::Skip, + ThreadCount::Inline, + ) + .unwrap(); + assert!(errors.is_empty(), "unexpected type errors: {errors:#?}"); + assert!(matches!(status, CommandExitStatus::Success)); + + let index = scip::types::Index::parse_from_bytes(&fs::read(&scip_path).unwrap()).unwrap(); + let metadata = index.metadata.unwrap(); + assert_eq!(metadata.tool_info.name, "pyrefly"); + assert_eq!(metadata.tool_info.version, "test"); + assert!( + metadata.project_root.starts_with("file://") + && metadata.project_root.ends_with("/project/"), + "unexpected project root: {}", + metadata.project_root + ); + } + #[test] fn incremental_check_commits_after_glean_loads_source_dependency() { let temp = TempDir::new().unwrap(); diff --git a/pyrefly/lib/report.rs b/pyrefly/lib/report.rs index ea5b3bf323..fe40d5a372 100644 --- a/pyrefly/lib/report.rs +++ b/pyrefly/lib/report.rs @@ -11,4 +11,5 @@ pub mod debug_info; pub mod dependency_graph; pub mod glean; pub mod pysa; +pub mod scip; pub mod trace; diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs new file mode 100644 index 0000000000..7bc48e9c71 --- /dev/null +++ b/pyrefly/lib/report/scip.rs @@ -0,0 +1,36 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::path::Path; + +use lsp_types::Uri; +use pyrefly_util::absolutize::Absolutize; +use scip::types::Index; +use scip::types::Metadata; +use scip::types::TextEncoding; +use scip::types::ToolInfo; + +pub fn scip(project_root: &Path, version: &str) -> Index { + let project_root = Uri::from_directory_path(project_root.absolutize()) + .map(|uri| uri.to_string()) + .unwrap_or_default(); + Index { + metadata: Some(Metadata { + project_root, + tool_info: Some(ToolInfo { + name: "pyrefly".to_owned(), + version: version.to_owned(), + ..Default::default() + }) + .into(), + text_document_encoding: TextEncoding::UTF8.into(), + ..Default::default() + }) + .into(), + ..Default::default() + } +} From b4d37b3563e4f595e34e2727e7c92bf69b983a6d Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sat, 17 Oct 2026 20:49:51 -0400 Subject: [PATCH 02/21] port over snapshot tests --- .cargo/config.toml | 1 + pyrefly/lib/commands/check.rs | 11 +- pyrefly/lib/report/scip.rs | 222 ++++++++++- pyrefly/lib/report/scip/snapshot.rs | 373 ++++++++++++++++++ .../snapshots/input/aliased_import/actual.py | 4 + .../snapshots/input/aliased_import/aliased.py | 1 + .../input/builtin_imports/builtin_imports.py | 6 + .../input/class_nohint/class_nohint.py | 14 + .../snapshots/input/comprehensions/comp.py | 16 + .../snapshots/input/dunder_vars/__main__.py | 4 + .../input/empty_project_name_version/main.py | 4 + .../input/enclosing_range_class/advanced.py | 13 + .../input/enclosing_range_class/simple.py | 3 + .../enclosing_range_function/advanced.py | 11 + .../input/enclosing_range_function/simple.py | 3 + .../scip/snapshots/input/f_string/fstring.py | 3 + .../input/file_from_module_import/abc/file.py | 3 + .../file_from_module.py | 3 + .../file_from_module_import/xyz/__init__.py | 0 .../xyz/nested_file.py | 1 + .../input/nameparts_import/nameparts.py | 4 + .../input/nested_items/src/__init__.py | 0 .../input/nested_items/src/foo/__init__.py | 0 .../nested_items/src/foo/bar/__init__.py | 2 + .../input/nested_items/src/foo/bar/baz/mod.py | 5 + .../input/nested_items/src/importer.py | 6 + .../input/nested_items/src/long_importer.py | 3 + .../snapshots/input/odd_pkg_name_1/main.py | 4 + .../scip/snapshots/input/pyproject_1/main.py | 4 + .../input/pyproject_1/pyproject.toml | 5 + .../snapshots/input/relative_import/bar.py | 7 + .../snapshots/input/relative_import/foo.py | 13 + .../input/single_class/src/single_class.py | 13 + .../single_function/src/single_function.py | 13 + .../input/unique/builtin_import_refs.py | 3 + .../snapshots/input/unique/field_docstring.py | 6 + .../input/unique/grandparent_impl.py | 10 + .../input/unique/inferred_field_docstring.py | 5 + .../snapshots/input/unique/inherits_class.py | 16 + .../input/unique/module_docstring.py | 8 + .../input/unique/multiinherits_test.py | 26 ++ .../snapshots/input/unique/property_access.py | 17 + .../input/unique/vars_inside_scopes.py | 11 + .../scip/snapshots/input/unique/walrus.py | 27 ++ .../input/unresolved_import/unresolved.py | 5 + .../snapshots/output/aliased_import/actual.py | 7 + .../output/aliased_import/aliased.py | 4 + .../output/builtin_imports/builtin_imports.py | 9 + .../output/class_nohint/class_nohint.py | 17 + .../snapshots/output/comprehensions/comp.py | 19 + .../snapshots/output/dunder_vars/__main__.py | 7 + .../output/empty_project_name_version/main.py | 7 + .../output/enclosing_range_class/advanced.py | 16 + .../output/enclosing_range_class/simple.py | 6 + .../enclosing_range_function/advanced.py | 14 + .../output/enclosing_range_function/simple.py | 6 + .../scip/snapshots/output/f_string/fstring.py | 6 + .../file_from_module_import/abc/file.py | 6 + .../file_from_module.py | 6 + .../file_from_module_import/xyz/__init__.py | 3 + .../xyz/nested_file.py | 4 + .../output/nameparts_import/nameparts.py | 7 + .../output/nested_items/src/__init__.py | 3 + .../output/nested_items/src/foo/__init__.py | 3 + .../nested_items/src/foo/bar/__init__.py | 5 + .../nested_items/src/foo/bar/baz/mod.py | 8 + .../output/nested_items/src/importer.py | 9 + .../output/nested_items/src/long_importer.py | 6 + .../snapshots/output/odd_pkg_name_1/main.py | 7 + .../scip/snapshots/output/pyproject_1/main.py | 7 + .../snapshots/output/relative_import/bar.py | 10 + .../snapshots/output/relative_import/foo.py | 16 + .../output/single_class/src/single_class.py | 16 + .../single_function/src/single_function.py | 16 + .../output/unique/builtin_import_refs.py | 6 + .../output/unique/field_docstring.py | 9 + .../output/unique/grandparent_impl.py | 13 + .../output/unique/inferred_field_docstring.py | 8 + .../snapshots/output/unique/inherits_class.py | 19 + .../output/unique/module_docstring.py | 11 + .../output/unique/multiinherits_test.py | 29 ++ .../output/unique/property_access.py | 20 + .../output/unique/vars_inside_scopes.py | 14 + .../scip/snapshots/output/unique/walrus.py | 30 ++ .../output/unresolved_import/unresolved.py | 8 + .../report/scip/snapshots/packageInfo.json | 16 + 86 files changed, 1336 insertions(+), 6 deletions(-) create mode 100644 pyrefly/lib/report/scip/snapshot.rs create mode 100644 pyrefly/lib/report/scip/snapshots/input/aliased_import/actual.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/aliased_import/aliased.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/builtin_imports/builtin_imports.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/class_nohint/class_nohint.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/comprehensions/comp.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/dunder_vars/__main__.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/empty_project_name_version/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/advanced.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/simple.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/advanced.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/simple.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/f_string/fstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/file_from_module_import/abc/file.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/file_from_module_import/file_from_module.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/file_from_module_import/xyz/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/file_from_module_import/xyz/nested_file.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nameparts_import/nameparts.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nested_items/src/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/baz/mod.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nested_items/src/importer.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nested_items/src/long_importer.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/odd_pkg_name_1/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/pyproject_1/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/pyproject_1/pyproject.toml create mode 100644 pyrefly/lib/report/scip/snapshots/input/relative_import/bar.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/relative_import/foo.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/single_class/src/single_class.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/single_function/src/single_function.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/builtin_import_refs.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/field_docstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/grandparent_impl.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/inferred_field_docstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/inherits_class.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/module_docstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/multiinherits_test.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/property_access.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/vars_inside_scopes.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/walrus.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/unresolved_import/unresolved.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/property_access.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/walrus.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py create mode 100644 pyrefly/lib/report/scip/snapshots/packageInfo.json diff --git a/.cargo/config.toml b/.cargo/config.toml index 140eef8fd5..9604f961dc 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -11,6 +11,7 @@ CINDERX_FIXTURES_PATH = { value = "pyrefly/lib/test/cinderx/fixtures", relative MARSHMALLOW_TEST_PATH = { value = "pyrefly/lib/test/marshmallow/third-party", relative = true } POLARS_TEST_PATH = { value = "pyrefly/lib/test/polars/third-party", relative = true } GLEAN_SNAPSHOTS_PATH = { value = "pyrefly/lib/report/glean/snapshots", relative = true } +SCIP_SNAPSHOTS_PATH = { value = "pyrefly/lib/report/scip/snapshots", relative = true } COVERAGE_TEST_PATH = { value = "pyrefly/lib/test/coverage/test_files", relative = true } STUBGEN_TEST_PATH = { value = "pyrefly/lib/test/stubgen", relative = true } SHAPE_EXTENSIONS_TEST_PATH = { value = "tensor-shapes/pyrefly-shape-extensions", relative = true } diff --git a/pyrefly/lib/commands/check.rs b/pyrefly/lib/commands/check.rs index 6117ae956b..10fe708bc2 100644 --- a/pyrefly/lib/commands/check.rs +++ b/pyrefly/lib/commands/check.rs @@ -2272,7 +2272,14 @@ impl CheckArgs { if let Some(path) = &self.output.report_scip { fs_anyhow::write( path, - report::scip::scip(&relative_to, version).write_to_bytes()?, + report::scip::scip( + transaction, + handles, + &relative_to, + &report::scip::Package::default(), + version, + ) + .write_to_bytes()?, )?; } if let Some(pysa_reporter) = transaction.take_pysa_reporter() { @@ -2593,6 +2600,8 @@ def go(w: Widget) -> int: assert!(matches!(status, CommandExitStatus::Success)); let index = scip::types::Index::parse_from_bytes(&fs::read(&scip_path).unwrap()).unwrap(); + assert_eq!(index.documents.len(), 1); + assert_eq!(index.documents[0].relative_path, "main.py"); let metadata = index.metadata.unwrap(); assert_eq!(metadata.tool_info.name, "pyrefly"); assert_eq!(metadata.tool_info.version, "test"); diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 7bc48e9c71..0131dc50a0 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -8,19 +8,55 @@ use std::path::Path; use lsp_types::Uri; +use pyrefly_build::handle::Handle; +use pyrefly_python::module_name::ModuleName; use pyrefly_util::absolutize::Absolutize; +use pyrefly_util::unix_path::path_to_unix_string; +use scip::symbol::format_symbol; +use scip::types::Descriptor; +use scip::types::Document; use scip::types::Index; use scip::types::Metadata; +use scip::types::Occurrence; +use scip::types::PositionEncoding; +use scip::types::Symbol; +use scip::types::SymbolRole; use scip::types::TextEncoding; use scip::types::ToolInfo; +use scip::types::descriptor::Suffix; -pub fn scip(project_root: &Path, version: &str) -> Index { - let project_root = Uri::from_directory_path(project_root.absolutize()) - .map(|uri| uri.to_string()) - .unwrap_or_default(); +use crate::state::state::Transaction; + +#[cfg(test)] +mod snapshot; + +const SCHEME: &str = "scip-python"; +const MANAGER: &str = "python"; + +#[derive(Debug, Clone, Default)] +pub struct Package { + pub name: String, + pub version: String, +} + +pub fn scip( + transaction: &Transaction, + handles: &[Handle], + project_root: &Path, + project: &Package, + version: &str, +) -> Index { + let project_root = project_root.absolutize(); + let mut documents: Vec = handles + .iter() + .filter_map(|handle| document(transaction, handle, &project_root, project)) + .collect(); + documents.sort_by(|a, b| a.relative_path.cmp(&b.relative_path)); Index { metadata: Some(Metadata { - project_root, + project_root: Uri::from_directory_path(&project_root) + .map(|uri| uri.to_string()) + .unwrap_or_default(), tool_info: Some(ToolInfo { name: "pyrefly".to_owned(), version: version.to_owned(), @@ -31,6 +67,182 @@ pub fn scip(project_root: &Path, version: &str) -> Index { ..Default::default() }) .into(), + documents, + ..Default::default() + } +} + +fn document( + transaction: &Transaction, + handle: &Handle, + project_root: &Path, + project: &Package, +) -> Option { + let module = transaction.get_module_info(handle)?; + let relative_path = module.path().as_path().strip_prefix(project_root).ok()?; + Some(Document { + language: "python".to_owned(), + relative_path: path_to_unix_string(relative_path), + position_encoding: PositionEncoding::UTF8CodeUnitOffsetFromLineStart.into(), + occurrences: vec![Occurrence { + range: vec![0, 0, 0], + symbol: module_symbol(project, module.name()), + symbol_roles: SymbolRole::Definition as i32, + ..Default::default() + }], + ..Default::default() + }) +} + +fn descriptor(name: &str, suffix: Suffix) -> Descriptor { + Descriptor { + name: name.to_owned(), + suffix: suffix.into(), + ..Default::default() + } +} + +fn global_symbol(package: &Package, descriptors: Vec) -> String { + // The SCIP spec escapes spaces in package fields by doubling them. + let escape = |x: &str| x.replace(' ', " "); + format_symbol(Symbol { + scheme: SCHEME.to_owned(), + package: Some(scip::types::Package { + manager: MANAGER.to_owned(), + name: escape(&package.name), + version: escape(&package.version), + ..Default::default() + }) + .into(), + descriptors, ..Default::default() + }) +} + +fn module_symbol(package: &Package, module: ModuleName) -> String { + global_symbol( + package, + vec![ + descriptor(module.as_str(), Suffix::Namespace), + descriptor("__init__", Suffix::Meta), + ], + ) +} + +#[cfg(test)] +mod tests { + use std::path::PathBuf; + + use pyrefly_python::module_path::ModulePath; + use pyrefly_util::fs_anyhow; + + use super::*; + use crate::report::scip::snapshot::format_snapshot; + use crate::state::require::Require; + use crate::test::util::TestEnv; + + fn snapshots_dir() -> PathBuf { + let path = std::env::var("SCIP_SNAPSHOTS_PATH") + .expect("SCIP_SNAPSHOTS_PATH env var not set: buck should set this automatically"); + std::env::current_dir().unwrap().join(path) } + + fn project_package(snapshots: &Path, case: &str) -> Package { + let info: serde_json::Value = serde_json::from_str( + &fs_anyhow::read_to_string(&snapshots.join("packageInfo.json")).unwrap(), + ) + .unwrap(); + let package = info["special"].get(case).unwrap_or(&info["default"]); + Package { + name: package["name"].as_str().unwrap().to_owned(), + version: package["version"].as_str().unwrap().to_owned(), + } + } + + fn assert_snapshot(case: &str) { + let snapshots = snapshots_dir(); + let input_dir = snapshots.join("input").join(case); + let output_dir = snapshots.join("output").join(case); + + let mut env = TestEnv::new(); + let mut handles = Vec::new(); + for path in glob::glob(&format!("{}/**/*.py", input_dir.display())).unwrap() { + let path = path.unwrap(); + let name = + ModuleName::from_relative_path(path.strip_prefix(&input_dir).unwrap()).unwrap(); + env.add_real_path(name.as_str(), path.clone()); + handles.push(Handle::new( + name, + ModulePath::filesystem(path), + env.sys_info(), + )); + } + let (state, _) = env + .with_default_require_level(Require::Everything) + .to_state(); + let index = scip( + &state.transaction(), + &handles, + &input_dir, + &project_package(&snapshots, case), + "test", + ); + + let update = std::env::var("SCIP_UPDATE_SNAPSHOTS").is_ok(); + for doc in &index.documents { + let input = fs_anyhow::read_to_string(&input_dir.join(&doc.relative_path)).unwrap(); + let actual = format_snapshot(&input, doc, &index.external_symbols); + let output_path = output_dir.join(&doc.relative_path); + if update { + fs_anyhow::create_dir_all(output_path.parent().unwrap()).unwrap(); + fs_anyhow::write(&output_path, actual).unwrap(); + continue; + } + let expected = fs_anyhow::read_to_string(&output_path).unwrap_or_else(|_| { + panic!( + "{} does not exist. Run with SCIP_UPDATE_SNAPSHOTS=1 to generate it.", + output_path.display() + ) + }); + pretty_assertions::assert_eq!( + expected, + actual, + "Snapshot mismatch for {}. Run with SCIP_UPDATE_SNAPSHOTS=1 to update.", + output_path.display() + ); + } + } + + macro_rules! snapshot_tests { + ($($case:ident),* $(,)?) => { + $( + #[test] + fn $case() { + assert_snapshot(stringify!($case)); + } + )* + }; + } + + snapshot_tests!( + aliased_import, + builtin_imports, + class_nohint, + comprehensions, + dunder_vars, + empty_project_name_version, + enclosing_range_class, + enclosing_range_function, + f_string, + file_from_module_import, + nameparts_import, + nested_items, + odd_pkg_name_1, + pyproject_1, + relative_import, + single_class, + single_function, + unique, + unresolved_import, + ); } diff --git a/pyrefly/lib/report/scip/snapshot.rs b/pyrefly/lib/report/scip/snapshot.rs new file mode 100644 index 0000000000..57975ba097 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshot.rs @@ -0,0 +1,373 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +//! A port of scip-python's `formatSnapshot` (`packages/pyright-scip/src/lib.ts`), so that +//! snapshots ported from scip-python keep the same layout. + +use std::collections::HashMap; +use std::collections::HashSet; + +use scip::types::Document; +use scip::types::Relationship; +use scip::types::SymbolInformation; +use scip::types::SymbolRole; + +use crate::report::scip::MANAGER; +use crate::report::scip::SCHEME; + +const FORMAT_OPTIONS_PREFIX: &str = "# format-options:"; + +#[derive(Default)] +struct FormatOptions { + show_docs: bool, + show_ranges: bool, +} + +impl FormatOptions { + fn parse(lines: &[&str]) -> Self { + let mut options = Self::default(); + if let Some(line) = lines + .iter() + .find_map(|line| line.strip_prefix(FORMAT_OPTIONS_PREFIX)) + { + for option in line.trim().split(',') { + match option.trim() { + "showDocs" => options.show_docs = true, + "showRanges" => options.show_ranges = true, + option => panic!("Invalid format option: {option}"), + } + } + } + options + } +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Position { + line: usize, + character: usize, +} + +#[derive(Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +struct Range { + start: Position, + end: Position, +} + +impl Range { + fn from_scip(range: &[i32]) -> Self { + let (end_line, end_character) = if range.len() == 3 { + (range[0], range[2]) + } else { + (range[2], range[3]) + }; + Self { + start: Position { + line: range[0] as usize, + character: range[1] as usize, + }, + end: Position { + line: end_line as usize, + character: end_character as usize, + }, + } + } +} + +fn is_definition(roles: i32) -> bool { + roles & SymbolRole::Definition as i32 != 0 +} + +struct EnclosingRange<'a> { + range: Range, + symbol: &'a str, +} + +struct Formatter<'a> { + out: String, + options: FormatOptions, + symbols: HashMap<&'a str, &'a SymbolInformation>, + external_symbols: HashMap<&'a str, &'a SymbolInformation>, + symbols_with_definitions: HashSet<&'a str>, + emitted_docs: HashSet<&'a str>, +} + +impl<'a> Formatter<'a> { + fn push_doc(&mut self, range: Range, symbol: &'a str, definition: bool, start_of_line: bool) { + if self.emitted_docs.contains(symbol) + || (!definition && self.symbols_with_definitions.contains(symbol)) + { + self.out.push('\n'); + return; + } + self.emitted_docs.insert(symbol); + + let mut prefix = "\n#".to_owned(); + if !start_of_line { + prefix.push_str(&" ".repeat(range.start.character.saturating_sub(1))); + } + let (info, external) = match self.external_symbols.get(symbol) { + Some(info) => (Some(*info), true), + None => (self.symbols.get(symbol).copied(), false), + }; + if let Some(info) = info { + self.push_docs(&prefix, &info.documentation, external); + self.push_relationships(&prefix, &info.relationships); + } + self.out.push('\n'); + } + + fn push_docs(&mut self, prefix: &str, docs: &[String], external: bool) { + if !self.options.show_docs { + return; + } + for doc in docs { + for (i, line) in doc.split('\n').enumerate() { + self.out.push_str(prefix); + if i == 0 { + if external { + self.out.push_str("external "); + } + self.out.push_str("documentation "); + } else { + self.out.push_str(" > "); + } + // scip-python truncates by UTF-16 code unit; chars are close enough here. + let truncated: String = line.chars().take(40).collect(); + self.out.push_str(&truncated); + if line.chars().count() > 40 { + self.out.push_str("..."); + } + } + } + } + + fn push_relationships(&mut self, prefix: &str, relationships: &[Relationship]) { + let mut relationships: Vec<_> = relationships.iter().collect(); + relationships.sort_by(|a, b| a.symbol.cmp(&b.symbol)); + for relationship in relationships { + self.out.push_str(prefix); + self.out.push_str("relationship"); + if relationship.is_implementation { + self.out.push_str(" implementation"); + } + if relationship.is_reference { + self.out.push_str(" reference"); + } + if relationship.is_type_definition { + self.out.push_str(" type_definition"); + } + self.out.push(' '); + self.out.push_str(&relationship.symbol); + } + } + + fn push_enclosing_range(&mut self, enclosing: &EnclosingRange, end: bool) { + let character = enclosing.range.start.character; + self.out.push('#'); + self.out + .push_str(&" ".repeat(character.saturating_sub(1).max(1))); + self.out.push(if character < 2 { + '<' + } else if end { + '^' + } else { + '⌄' + }); + self.out.push_str(if end { " end " } else { " start " }); + self.out.push_str("enclosing_range "); + self.out.push_str(enclosing.symbol); + self.out.push('\n'); + } +} + +pub fn format_snapshot( + input: &str, + doc: &Document, + external_symbols: &[SymbolInformation], +) -> String { + let lines: Vec<&str> = input.split('\n').collect(); + let mut formatter = Formatter { + out: String::new(), + options: FormatOptions::parse(&lines), + symbols: doc.symbols.iter().map(|s| (s.symbol.as_str(), s)).collect(), + external_symbols: external_symbols + .iter() + .map(|s| (s.symbol.as_str(), s)) + .collect(), + symbols_with_definitions: HashSet::new(), + emitted_docs: HashSet::new(), + }; + + let mut enclosing_ranges = Vec::new(); + for occurrence in &doc.occurrences { + if is_definition(occurrence.symbol_roles) { + formatter + .symbols_with_definitions + .insert(occurrence.symbol.as_str()); + } + if formatter.options.show_ranges && !occurrence.enclosing_range.is_empty() { + enclosing_ranges.push(EnclosingRange { + range: Range::from_scip(&occurrence.enclosing_range), + symbol: &occurrence.symbol, + }); + } + } + // The range that starts first, and for equal starts, the one that ends last. + enclosing_ranges.sort_by(|a, b| { + a.range + .cmp(&b.range) + .then(b.range.end.line.cmp(&a.range.end.line)) + }); + let mut starts: Vec> = (0..lines.len()).map(|_| Vec::new()).collect(); + let mut ends: Vec> = (0..lines.len()).map(|_| Vec::new()).collect(); + for enclosing in &enclosing_ranges { + starts[enclosing.range.start.line].push(enclosing); + ends[enclosing.range.end.line].insert(0, enclosing); + } + + let mut occurrences: Vec<_> = doc.occurrences.iter().collect(); + occurrences.sort_by_key(|o| Range::from_scip(&o.range)); + let package_prefix = format!("{SCHEME} {MANAGER}"); + let mut index = 0; + + for (line_number, line) in lines.iter().enumerate() { + // A zero-width occurrence at the start of the file is a file-level item, written above + // the first line. + if index == 0 + && let Some(occurrence) = occurrences.first() + { + let range = Range::from_scip(&occurrence.range); + if range.start.character == 0 && range.end.character == 0 { + let definition = is_definition(occurrence.symbol_roles); + formatter.out.push_str("# < "); + formatter.out.push_str(if definition { + "definition" + } else { + "reference" + }); + formatter.out.push(' '); + formatter.out.push_str(&occurrence.symbol); + formatter.push_doc(range, &occurrence.symbol, definition, true); + formatter.out.push('\n'); + index += 1; + } + } + + if formatter.options.show_ranges { + for enclosing in &starts[line_number] { + formatter.push_enclosing_range(enclosing, false); + } + } + + formatter.out.push_str(line); + formatter.out.push('\n'); + while let Some(occurrence) = occurrences.get(index) + && occurrence.range[0] as usize == line_number + { + index += 1; + assert!( + occurrence.range.len() == 3, + "multi-line occurrence ranges are not supported" + ); + let range = Range::from_scip(&occurrence.range); + let start_of_line = range.start.character == 0; + formatter.out.push('#'); + if !start_of_line { + formatter + .out + .push_str(&" ".repeat(range.start.character - 1)); + } + let caret_length = (range.end.character - range.start.character) + .checked_sub(start_of_line as usize) + .expect("negative length occurrence"); + formatter.out.push_str(&"^".repeat(caret_length)); + let definition = is_definition(occurrence.symbol_roles); + formatter.out.push_str(if definition { + " definition " + } else { + " reference " + }); + let symbol = occurrence + .symbol + .strip_prefix(&package_prefix) + .unwrap_or(&occurrence.symbol); + formatter.out.push_str(&symbol.replacen('\n', "|", 1)); + formatter.push_doc(range, &occurrence.symbol, definition, start_of_line); + } + + if formatter.options.show_ranges { + for enclosing in &ends[line_number] { + formatter.push_enclosing_range(enclosing, true); + } + } + } + formatter.out +} + +#[cfg(test)] +mod tests { + use scip::types::Occurrence; + + use super::*; + + fn occurrence(range: Vec, symbol: &str, roles: i32, enclosing: Vec) -> Occurrence { + Occurrence { + range, + symbol: symbol.to_owned(), + symbol_roles: roles, + enclosing_range: enclosing, + ..Default::default() + } + } + + #[test] + fn test_docs_relationships_and_ranges() { + let input = "# format-options: showDocs, showRanges\nclass C:\n def f(self):\n return self\n"; + let def = SymbolRole::Definition as i32; + let module = "scip-python python p 1 m/__init__:"; + let f = "scip-python python p 1 m/C#f()."; + let param = "scip-python python p 1 m/C#f().(self)"; + let doc = Document { + occurrences: vec![ + occurrence(vec![0, 0, 0], module, def, vec![]), + occurrence(vec![2, 8, 9], f, def, vec![2, 4, 3, 19]), + occurrence(vec![2, 10, 14], param, def, vec![]), + occurrence(vec![3, 15, 19], param, 0, vec![]), + ], + symbols: vec![SymbolInformation { + symbol: f.to_owned(), + documentation: vec!["```python\ndef f(self)\n```".to_owned()], + relationships: vec![Relationship { + symbol: "scip-python python p 1 m/B#f().".to_owned(), + is_implementation: true, + ..Default::default() + }], + ..Default::default() + }], + ..Default::default() + }; + let expected = r#"# < definition scip-python python p 1 m/__init__: + +# format-options: showDocs, showRanges +class C: +# ⌄ start enclosing_range scip-python python p 1 m/C#f(). + def f(self): +# ^ definition p 1 m/C#f(). +# documentation ```python +# > def f(self) +# > ``` +# relationship implementation scip-python python p 1 m/B#f(). +# ^^^^ definition p 1 m/C#f().(self) + return self +# ^^^^ reference p 1 m/C#f().(self) +# ^ end enclosing_range scip-python python p 1 m/C#f(). + +"#; + pretty_assertions::assert_eq!(expected, format_snapshot(input, &doc, &[])); + } +} diff --git a/pyrefly/lib/report/scip/snapshots/input/aliased_import/actual.py b/pyrefly/lib/report/scip/snapshots/input/aliased_import/actual.py new file mode 100644 index 0000000000..ad06034c82 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/aliased_import/actual.py @@ -0,0 +1,4 @@ +import aliased +import aliased as A + +print(A.SOME_CONSTANT) diff --git a/pyrefly/lib/report/scip/snapshots/input/aliased_import/aliased.py b/pyrefly/lib/report/scip/snapshots/input/aliased_import/aliased.py new file mode 100644 index 0000000000..0c0a66160a --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/aliased_import/aliased.py @@ -0,0 +1 @@ +SOME_CONSTANT = "Some Constant" diff --git a/pyrefly/lib/report/scip/snapshots/input/builtin_imports/builtin_imports.py b/pyrefly/lib/report/scip/snapshots/input/builtin_imports/builtin_imports.py new file mode 100644 index 0000000000..b6bea9e794 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/builtin_imports/builtin_imports.py @@ -0,0 +1,6 @@ +# format-options: showDocs + +import re +from typing import Callable, Optional + +print(re, Callable, Optional) diff --git a/pyrefly/lib/report/scip/snapshots/input/class_nohint/class_nohint.py b/pyrefly/lib/report/scip/snapshots/input/class_nohint/class_nohint.py new file mode 100644 index 0000000000..8ae5ca8829 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/class_nohint/class_nohint.py @@ -0,0 +1,14 @@ +# format-options: showDocs + +class Example: + # Note, only y has a type hint + y: int + + def __init__(self, in_val): + self.x = in_val + self.x = self.x + 1 + self.y = in_val + + def something(self): + print(self.x) + print(self.y) diff --git a/pyrefly/lib/report/scip/snapshots/input/comprehensions/comp.py b/pyrefly/lib/report/scip/snapshots/input/comprehensions/comp.py new file mode 100644 index 0000000000..7d85f23a8b --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/comprehensions/comp.py @@ -0,0 +1,16 @@ +def something(x): + return 2 * x + +_ = [x for x in "should be local 0"] +_ = [something(x) for x in "should be local 1"] +_ = [something(x) for x in "should be local 2" if x == "s"] +_ = {k: x for (k, x) in enumerate(range(10))} + + +asdf = (var for var in [1, 2, 3] if var % 2 == 0) +for var in asdf: + print(var) + +# TODO: ListComprehensions with if +# TODO: Dictionary comprehensions + diff --git a/pyrefly/lib/report/scip/snapshots/input/dunder_vars/__main__.py b/pyrefly/lib/report/scip/snapshots/input/dunder_vars/__main__.py new file mode 100644 index 0000000000..5bd2e47d79 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/dunder_vars/__main__.py @@ -0,0 +1,4 @@ + +if __name__ == '__main__': + print("main") + diff --git a/pyrefly/lib/report/scip/snapshots/input/empty_project_name_version/main.py b/pyrefly/lib/report/scip/snapshots/input/empty_project_name_version/main.py new file mode 100644 index 0000000000..b4a3ff496a --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/empty_project_name_version/main.py @@ -0,0 +1,4 @@ +def main(): + pass + +main() diff --git a/pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/advanced.py b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/advanced.py new file mode 100644 index 0000000000..abdc18fcb1 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/advanced.py @@ -0,0 +1,13 @@ +# format-options: showRanges +def class_decorator(cls): + def wrapper(*args, **kwargs): + return cls(*args, **kwargs) + return wrapper + +@class_decorator +class Test: + def __init__(self, x: float): + self.x = x + + def test(self) -> float: + return self.x diff --git a/pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/simple.py b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/simple.py new file mode 100644 index 0000000000..26746956e9 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_class/simple.py @@ -0,0 +1,3 @@ +# format-options: showRanges +class Test(): + pass diff --git a/pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/advanced.py b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/advanced.py new file mode 100644 index 0000000000..5eff95e691 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/advanced.py @@ -0,0 +1,11 @@ +# format-options: showRanges +def decorator(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + +@decorator +def func(x: float) -> float: + test = x + + return test diff --git a/pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/simple.py b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/simple.py new file mode 100644 index 0000000000..3bededc3bf --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/enclosing_range_function/simple.py @@ -0,0 +1,3 @@ +# format-options: showRanges +def simple(x): + return x diff --git a/pyrefly/lib/report/scip/snapshots/input/f_string/fstring.py b/pyrefly/lib/report/scip/snapshots/input/f_string/fstring.py new file mode 100644 index 0000000000..8726cb9159 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/f_string/fstring.py @@ -0,0 +1,3 @@ +var = ", world!" + +print(f"var: hello {var}") diff --git a/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/abc/file.py b/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/abc/file.py new file mode 100644 index 0000000000..4d523c10eb --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/abc/file.py @@ -0,0 +1,3 @@ +from xyz import nested_file + +print(nested_file.X) diff --git a/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/file_from_module.py b/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/file_from_module.py new file mode 100644 index 0000000000..4d523c10eb --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/file_from_module.py @@ -0,0 +1,3 @@ +from xyz import nested_file + +print(nested_file.X) diff --git a/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/xyz/__init__.py b/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/xyz/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/xyz/nested_file.py b/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/xyz/nested_file.py new file mode 100644 index 0000000000..5f4d995029 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/file_from_module_import/xyz/nested_file.py @@ -0,0 +1 @@ +X = "a constant" diff --git a/pyrefly/lib/report/scip/snapshots/input/nameparts_import/nameparts.py b/pyrefly/lib/report/scip/snapshots/input/nameparts_import/nameparts.py new file mode 100644 index 0000000000..ca52781502 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/nameparts_import/nameparts.py @@ -0,0 +1,4 @@ +import importlib.resources + +importlib.resources.read_text('pre_commit.resources', 'filename') +importlib.resources.read_text('pre_commit.resources', 'filename') diff --git a/pyrefly/lib/report/scip/snapshots/input/nested_items/src/__init__.py b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/__init__.py b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/__init__.py b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/__init__.py new file mode 100644 index 0000000000..174374a284 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/__init__.py @@ -0,0 +1,2 @@ +class InitClass: + init_item = 10 diff --git a/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/baz/mod.py b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/baz/mod.py new file mode 100644 index 0000000000..86ea18d7c8 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/foo/bar/baz/mod.py @@ -0,0 +1,5 @@ +class SuchNestedMuchWow: + class_item: int = 42 + +class AnotherNestedMuchWow: + other_item: int = 42 diff --git a/pyrefly/lib/report/scip/snapshots/input/nested_items/src/importer.py b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/importer.py new file mode 100644 index 0000000000..c43986f8cd --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/importer.py @@ -0,0 +1,6 @@ +from foo.bar import InitClass +from foo.bar.baz.mod import SuchNestedMuchWow, AnotherNestedMuchWow + +print(SuchNestedMuchWow().class_item) +print(AnotherNestedMuchWow().other_item) +print(InitClass().init_item) diff --git a/pyrefly/lib/report/scip/snapshots/input/nested_items/src/long_importer.py b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/long_importer.py new file mode 100644 index 0000000000..bcfde5e76c --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/nested_items/src/long_importer.py @@ -0,0 +1,3 @@ +import foo.bar.baz.mod + +print(foo.bar.baz.mod.SuchNestedMuchWow) diff --git a/pyrefly/lib/report/scip/snapshots/input/odd_pkg_name_1/main.py b/pyrefly/lib/report/scip/snapshots/input/odd_pkg_name_1/main.py new file mode 100644 index 0000000000..1cfbad29fa --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/odd_pkg_name_1/main.py @@ -0,0 +1,4 @@ +def main(): + pass + +main() diff --git a/pyrefly/lib/report/scip/snapshots/input/pyproject_1/main.py b/pyrefly/lib/report/scip/snapshots/input/pyproject_1/main.py new file mode 100644 index 0000000000..1cfbad29fa --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/pyproject_1/main.py @@ -0,0 +1,4 @@ +def main(): + pass + +main() diff --git a/pyrefly/lib/report/scip/snapshots/input/pyproject_1/pyproject.toml b/pyrefly/lib/report/scip/snapshots/input/pyproject_1/pyproject.toml new file mode 100644 index 0000000000..3e51bbfbc6 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/pyproject_1/pyproject.toml @@ -0,0 +1,5 @@ +[project] +name = "abc" + +[tool.poetry] +version = "16.05" diff --git a/pyrefly/lib/report/scip/snapshots/input/relative_import/bar.py b/pyrefly/lib/report/scip/snapshots/input/relative_import/bar.py new file mode 100644 index 0000000000..de5b84db49 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/relative_import/bar.py @@ -0,0 +1,7 @@ +from .foo import exported_function +from .foo import this_class + +if True: + exported_function() + + this_class.exported_function() diff --git a/pyrefly/lib/report/scip/snapshots/input/relative_import/foo.py b/pyrefly/lib/report/scip/snapshots/input/relative_import/foo.py new file mode 100644 index 0000000000..43398d1efe --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/relative_import/foo.py @@ -0,0 +1,13 @@ +def exported_function(): + return "function" + +class MyClass: + """This is a class and it is cool""" + + def __init__(self): + pass + + def exported_function(self): + return "exported" + +this_class = MyClass() diff --git a/pyrefly/lib/report/scip/snapshots/input/single_class/src/single_class.py b/pyrefly/lib/report/scip/snapshots/input/single_class/src/single_class.py new file mode 100644 index 0000000000..0646fcf957 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/single_class/src/single_class.py @@ -0,0 +1,13 @@ +class ExampleClass: + a: int + b: int + c: str + + static_var = "Hello World" + + def __init__(self, a: int, b: int): + local_c = ", world!" + + self.a = a + self.b = b + self.c = "hello" + local_c diff --git a/pyrefly/lib/report/scip/snapshots/input/single_function/src/single_function.py b/pyrefly/lib/report/scip/snapshots/input/single_function/src/single_function.py new file mode 100644 index 0000000000..5b2a5ef8e8 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/single_function/src/single_function.py @@ -0,0 +1,13 @@ +def my_cool_function(a: str) -> str: + x = ", world" + return a + x + +def my_cool_function_2(a: str): + x = ", world" + return (lambda y: a + x + y)("oh no") + +def next_level(): + return my_cool_function + +my_cool_function("hello") +next_level()(a = "wow") diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/builtin_import_refs.py b/pyrefly/lib/report/scip/snapshots/input/unique/builtin_import_refs.py new file mode 100644 index 0000000000..df4abf22f8 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/builtin_import_refs.py @@ -0,0 +1,3 @@ +from typing import Any + +print(Any) diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/field_docstring.py b/pyrefly/lib/report/scip/snapshots/input/unique/field_docstring.py new file mode 100644 index 0000000000..e7bd35227a --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/field_docstring.py @@ -0,0 +1,6 @@ +# format-options: showDocs + +class ClassWithField: + + a: int + """Hello world, this is a""" diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/grandparent_impl.py b/pyrefly/lib/report/scip/snapshots/input/unique/grandparent_impl.py new file mode 100644 index 0000000000..0b0add32c8 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/grandparent_impl.py @@ -0,0 +1,10 @@ +class A: + def grandparent(self) -> bool: + return True + +class B(A): + ... + +class C(B): + def grandparent(self) -> bool: + return False diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/inferred_field_docstring.py b/pyrefly/lib/report/scip/snapshots/input/unique/inferred_field_docstring.py new file mode 100644 index 0000000000..e1471fd570 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/inferred_field_docstring.py @@ -0,0 +1,5 @@ +# format-options: showDocs + +class ClassWithInferredField: + def __init__(self, b: int): + self.b = b diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/inherits_class.py b/pyrefly/lib/report/scip/snapshots/input/unique/inherits_class.py new file mode 100644 index 0000000000..0c27075555 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/inherits_class.py @@ -0,0 +1,16 @@ +class A: + def x(self) -> int: + raise NotImplemented + + def matched_despite_different_type(self, x: int): + pass + +class B(A): + def x(self) -> int: + return 5 + + def matched_despite_different_type(self, x: int, y: int): + pass + + def unrelated(self): + pass diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/module_docstring.py b/pyrefly/lib/report/scip/snapshots/input/unique/module_docstring.py new file mode 100644 index 0000000000..5d8a1ee205 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/module_docstring.py @@ -0,0 +1,8 @@ +# format-options: showDocs + +""" +This is a docstring for this module +""" + +def x(): + return 1 diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/multiinherits_test.py b/pyrefly/lib/report/scip/snapshots/input/unique/multiinherits_test.py new file mode 100644 index 0000000000..6eb4799ef5 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/multiinherits_test.py @@ -0,0 +1,26 @@ +class Left: + def one(self) -> int: + return 1 + + def shared(self) -> bool: + return False + +class Right: + def two(self): + return 2 + + def shared(self) -> bool: + return False + +class Multi(Left, Right): + def one(self) -> int: + return 1 + + def two(self): + return 2 + + def three(self): + return 3 + + def shared(self) -> bool: + return True diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/property_access.py b/pyrefly/lib/report/scip/snapshots/input/unique/property_access.py new file mode 100644 index 0000000000..d883c5b587 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/property_access.py @@ -0,0 +1,17 @@ +from typing import Sequence + +class PropertyClass: + def __init__(self): + pass + + @property + def prop_ref(self): + return 5 + + +xs = [PropertyClass()] + +def usage(xs: Sequence[PropertyClass]): + def nested(): + for x in xs: + print(x.prop_ref) diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/vars_inside_scopes.py b/pyrefly/lib/report/scip/snapshots/input/unique/vars_inside_scopes.py new file mode 100644 index 0000000000..418b7318d8 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/vars_inside_scopes.py @@ -0,0 +1,11 @@ +from typing import List + +class X: + items: List[int] + + def my_func(self): + for x in self.items: + y = x + 1 + + if 5 in self.items: + z = "oh ya" diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/walrus.py b/pyrefly/lib/report/scip/snapshots/input/unique/walrus.py new file mode 100644 index 0000000000..b21d31c65d --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/walrus.py @@ -0,0 +1,27 @@ +import re + +some_list = [1, 2, 3, 4, 5, 100, 1000] +pattern = "^[A-Za-z0-9_]*$" +text = "Some_name123" + +# if statement +if (n := len(some_list)) > 10: + print(f"List is too long with {n} elements.") + +if (match := re.search(pattern, text)) is not None: + print("Found match:", match.group(0)) + + +# comprehensions +def show_some_comprehension(): + [print(x, root) for x in some_list if (x % 2 == 0) and (root := x**0.5) > 5] + + +# while loop +while (line := input("Enter text: ")) != "quit": + print(f"You entered: {line}") + + +# if + comprehension +if any((any_n := num) < 0 for num in some_list): + print(f"Negative number found: {any_n}") diff --git a/pyrefly/lib/report/scip/snapshots/input/unresolved_import/unresolved.py b/pyrefly/lib/report/scip/snapshots/input/unresolved_import/unresolved.py new file mode 100644 index 0000000000..4747f7b96d --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unresolved_import/unresolved.py @@ -0,0 +1,5 @@ +import this_is_not_real + +print(this_is_not_real.x) +print(this_is_not_real.x) + diff --git a/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py new file mode 100644 index 0000000000..e4cb829184 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py @@ -0,0 +1,7 @@ +# < definition scip-python python snapshot-util 0.1 actual/__init__: + +import aliased +import aliased as A + +print(A.SOME_CONSTANT) + diff --git a/pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py b/pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py new file mode 100644 index 0000000000..853108f780 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py @@ -0,0 +1,4 @@ +# < definition scip-python python snapshot-util 0.1 aliased/__init__: + +SOME_CONSTANT = "Some Constant" + diff --git a/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py new file mode 100644 index 0000000000..e17dd7242d --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py @@ -0,0 +1,9 @@ +# < definition scip-python python snapshot-util 0.1 builtin_imports/__init__: + +# format-options: showDocs + +import re +from typing import Callable, Optional + +print(re, Callable, Optional) + diff --git a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py new file mode 100644 index 0000000000..a705912479 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py @@ -0,0 +1,17 @@ +# < definition scip-python python snapshot-util 0.1 class_nohint/__init__: + +# format-options: showDocs + +class Example: + # Note, only y has a type hint + y: int + + def __init__(self, in_val): + self.x = in_val + self.x = self.x + 1 + self.y = in_val + + def something(self): + print(self.x) + print(self.y) + diff --git a/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py new file mode 100644 index 0000000000..6b7a119fde --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py @@ -0,0 +1,19 @@ +# < definition scip-python python snapshot-util 0.1 comp/__init__: + +def something(x): + return 2 * x + +_ = [x for x in "should be local 0"] +_ = [something(x) for x in "should be local 1"] +_ = [something(x) for x in "should be local 2" if x == "s"] +_ = {k: x for (k, x) in enumerate(range(10))} + + +asdf = (var for var in [1, 2, 3] if var % 2 == 0) +for var in asdf: + print(var) + +# TODO: ListComprehensions with if +# TODO: Dictionary comprehensions + + diff --git a/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py b/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py new file mode 100644 index 0000000000..cb69e1c3cb --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py @@ -0,0 +1,7 @@ +# < definition scip-python python snapshot-util 0.1 __main__/__init__: + + +if __name__ == '__main__': + print("main") + + diff --git a/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py b/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py new file mode 100644 index 0000000000..fd7668a411 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py @@ -0,0 +1,7 @@ +# < definition scip-python python . . main/__init__: + +def main(): + pass + +main() + diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py new file mode 100644 index 0000000000..21afdceee3 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py @@ -0,0 +1,16 @@ +# < definition scip-python python snapshot-util 0.1 advanced/__init__: + +# format-options: showRanges +def class_decorator(cls): + def wrapper(*args, **kwargs): + return cls(*args, **kwargs) + return wrapper + +@class_decorator +class Test: + def __init__(self, x: float): + self.x = x + + def test(self) -> float: + return self.x + diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py new file mode 100644 index 0000000000..65703beeeb --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py @@ -0,0 +1,6 @@ +# < definition scip-python python snapshot-util 0.1 simple/__init__: + +# format-options: showRanges +class Test(): + pass + diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py new file mode 100644 index 0000000000..4d108f48f5 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py @@ -0,0 +1,14 @@ +# < definition scip-python python snapshot-util 0.1 advanced/__init__: + +# format-options: showRanges +def decorator(func): + def wrapper(*args, **kwargs): + return func(*args, **kwargs) + return wrapper + +@decorator +def func(x: float) -> float: + test = x + + return test + diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py new file mode 100644 index 0000000000..aef0aa63ff --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py @@ -0,0 +1,6 @@ +# < definition scip-python python snapshot-util 0.1 simple/__init__: + +# format-options: showRanges +def simple(x): + return x + diff --git a/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py b/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py new file mode 100644 index 0000000000..3a5a355b8e --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py @@ -0,0 +1,6 @@ +# < definition scip-python python snapshot-util 0.1 fstring/__init__: + +var = ", world!" + +print(f"var: hello {var}") + diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py new file mode 100644 index 0000000000..c05cd172e1 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py @@ -0,0 +1,6 @@ +# < definition scip-python python snapshot-util 0.1 `abc.file`/__init__: + +from xyz import nested_file + +print(nested_file.X) + diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py new file mode 100644 index 0000000000..223f9c47cb --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py @@ -0,0 +1,6 @@ +# < definition scip-python python snapshot-util 0.1 file_from_module/__init__: + +from xyz import nested_file + +print(nested_file.X) + diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/__init__.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/__init__.py new file mode 100644 index 0000000000..25308027e7 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/__init__.py @@ -0,0 +1,3 @@ +# < definition scip-python python snapshot-util 0.1 xyz/__init__: + + diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py new file mode 100644 index 0000000000..3beac87099 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py @@ -0,0 +1,4 @@ +# < definition scip-python python snapshot-util 0.1 `xyz.nested_file`/__init__: + +X = "a constant" + diff --git a/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py new file mode 100644 index 0000000000..ea00e7f72f --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py @@ -0,0 +1,7 @@ +# < definition scip-python python snapshot-util 0.1 nameparts/__init__: + +import importlib.resources + +importlib.resources.read_text('pre_commit.resources', 'filename') +importlib.resources.read_text('pre_commit.resources', 'filename') + diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py new file mode 100644 index 0000000000..00291ef7ae --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py @@ -0,0 +1,3 @@ +# < definition scip-python python snapshot-util 0.1 src/__init__: + + diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py new file mode 100644 index 0000000000..39b336df65 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py @@ -0,0 +1,3 @@ +# < definition scip-python python snapshot-util 0.1 `src.foo`/__init__: + + diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py new file mode 100644 index 0000000000..2b85a27b40 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py @@ -0,0 +1,5 @@ +# < definition scip-python python snapshot-util 0.1 `src.foo.bar`/__init__: + +class InitClass: + init_item = 10 + diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py new file mode 100644 index 0000000000..74f0bab410 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py @@ -0,0 +1,8 @@ +# < definition scip-python python snapshot-util 0.1 `src.foo.bar.baz.mod`/__init__: + +class SuchNestedMuchWow: + class_item: int = 42 + +class AnotherNestedMuchWow: + other_item: int = 42 + diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py new file mode 100644 index 0000000000..871db5e815 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py @@ -0,0 +1,9 @@ +# < definition scip-python python snapshot-util 0.1 `src.importer`/__init__: + +from foo.bar import InitClass +from foo.bar.baz.mod import SuchNestedMuchWow, AnotherNestedMuchWow + +print(SuchNestedMuchWow().class_item) +print(AnotherNestedMuchWow().other_item) +print(InitClass().init_item) + diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py new file mode 100644 index 0000000000..8c157c0059 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py @@ -0,0 +1,6 @@ +# < definition scip-python python snapshot-util 0.1 `src.long_importer`/__init__: + +import foo.bar.baz.mod + +print(foo.bar.baz.mod.SuchNestedMuchWow) + diff --git a/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py b/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py new file mode 100644 index 0000000000..85ba6304a5 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py @@ -0,0 +1,7 @@ +# < definition scip-python python package with space 0.1 main/__init__: + +def main(): + pass + +main() + diff --git a/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py new file mode 100644 index 0000000000..2a36ab316e --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py @@ -0,0 +1,7 @@ +# < definition scip-python python snapshot-util 0.1 main/__init__: + +def main(): + pass + +main() + diff --git a/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py new file mode 100644 index 0000000000..8811594b77 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py @@ -0,0 +1,10 @@ +# < definition scip-python python snapshot-util 0.1 bar/__init__: + +from .foo import exported_function +from .foo import this_class + +if True: + exported_function() + + this_class.exported_function() + diff --git a/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py b/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py new file mode 100644 index 0000000000..4023b5e63e --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py @@ -0,0 +1,16 @@ +# < definition scip-python python snapshot-util 0.1 foo/__init__: + +def exported_function(): + return "function" + +class MyClass: + """This is a class and it is cool""" + + def __init__(self): + pass + + def exported_function(self): + return "exported" + +this_class = MyClass() + diff --git a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py new file mode 100644 index 0000000000..e339e36518 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py @@ -0,0 +1,16 @@ +# < definition scip-python python snapshot-util 0.1 `src.single_class`/__init__: + +class ExampleClass: + a: int + b: int + c: str + + static_var = "Hello World" + + def __init__(self, a: int, b: int): + local_c = ", world!" + + self.a = a + self.b = b + self.c = "hello" + local_c + diff --git a/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py new file mode 100644 index 0000000000..372bb931fb --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py @@ -0,0 +1,16 @@ +# < definition scip-python python snapshot-util 0.1 `src.single_function`/__init__: + +def my_cool_function(a: str) -> str: + x = ", world" + return a + x + +def my_cool_function_2(a: str): + x = ", world" + return (lambda y: a + x + y)("oh no") + +def next_level(): + return my_cool_function + +my_cool_function("hello") +next_level()(a = "wow") + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py new file mode 100644 index 0000000000..10fb85dbf3 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py @@ -0,0 +1,6 @@ +# < definition scip-python python snapshot-util 0.1 builtin_import_refs/__init__: + +from typing import Any + +print(Any) + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py new file mode 100644 index 0000000000..0137516f42 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py @@ -0,0 +1,9 @@ +# < definition scip-python python snapshot-util 0.1 field_docstring/__init__: + +# format-options: showDocs + +class ClassWithField: + + a: int + """Hello world, this is a""" + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py new file mode 100644 index 0000000000..d35c1803ba --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py @@ -0,0 +1,13 @@ +# < definition scip-python python snapshot-util 0.1 grandparent_impl/__init__: + +class A: + def grandparent(self) -> bool: + return True + +class B(A): + ... + +class C(B): + def grandparent(self) -> bool: + return False + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py new file mode 100644 index 0000000000..574700506f --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py @@ -0,0 +1,8 @@ +# < definition scip-python python snapshot-util 0.1 inferred_field_docstring/__init__: + +# format-options: showDocs + +class ClassWithInferredField: + def __init__(self, b: int): + self.b = b + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py new file mode 100644 index 0000000000..d0e019f823 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py @@ -0,0 +1,19 @@ +# < definition scip-python python snapshot-util 0.1 inherits_class/__init__: + +class A: + def x(self) -> int: + raise NotImplemented + + def matched_despite_different_type(self, x: int): + pass + +class B(A): + def x(self) -> int: + return 5 + + def matched_despite_different_type(self, x: int, y: int): + pass + + def unrelated(self): + pass + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py new file mode 100644 index 0000000000..5ea528b192 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py @@ -0,0 +1,11 @@ +# < definition scip-python python snapshot-util 0.1 module_docstring/__init__: + +# format-options: showDocs + +""" +This is a docstring for this module +""" + +def x(): + return 1 + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py new file mode 100644 index 0000000000..3f890f4713 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py @@ -0,0 +1,29 @@ +# < definition scip-python python snapshot-util 0.1 multiinherits_test/__init__: + +class Left: + def one(self) -> int: + return 1 + + def shared(self) -> bool: + return False + +class Right: + def two(self): + return 2 + + def shared(self) -> bool: + return False + +class Multi(Left, Right): + def one(self) -> int: + return 1 + + def two(self): + return 2 + + def three(self): + return 3 + + def shared(self) -> bool: + return True + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py new file mode 100644 index 0000000000..f8395bb251 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py @@ -0,0 +1,20 @@ +# < definition scip-python python snapshot-util 0.1 property_access/__init__: + +from typing import Sequence + +class PropertyClass: + def __init__(self): + pass + + @property + def prop_ref(self): + return 5 + + +xs = [PropertyClass()] + +def usage(xs: Sequence[PropertyClass]): + def nested(): + for x in xs: + print(x.prop_ref) + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py new file mode 100644 index 0000000000..df2d0be2f3 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py @@ -0,0 +1,14 @@ +# < definition scip-python python snapshot-util 0.1 vars_inside_scopes/__init__: + +from typing import List + +class X: + items: List[int] + + def my_func(self): + for x in self.items: + y = x + 1 + + if 5 in self.items: + z = "oh ya" + diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py new file mode 100644 index 0000000000..1f01659f95 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py @@ -0,0 +1,30 @@ +# < definition scip-python python snapshot-util 0.1 walrus/__init__: + +import re + +some_list = [1, 2, 3, 4, 5, 100, 1000] +pattern = "^[A-Za-z0-9_]*$" +text = "Some_name123" + +# if statement +if (n := len(some_list)) > 10: + print(f"List is too long with {n} elements.") + +if (match := re.search(pattern, text)) is not None: + print("Found match:", match.group(0)) + + +# comprehensions +def show_some_comprehension(): + [print(x, root) for x in some_list if (x % 2 == 0) and (root := x**0.5) > 5] + + +# while loop +while (line := input("Enter text: ")) != "quit": + print(f"You entered: {line}") + + +# if + comprehension +if any((any_n := num) < 0 for num in some_list): + print(f"Negative number found: {any_n}") + diff --git a/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py b/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py new file mode 100644 index 0000000000..3d890deb1d --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py @@ -0,0 +1,8 @@ +# < definition scip-python python snapshot-util 0.1 unresolved/__init__: + +import this_is_not_real + +print(this_is_not_real.x) +print(this_is_not_real.x) + + diff --git a/pyrefly/lib/report/scip/snapshots/packageInfo.json b/pyrefly/lib/report/scip/snapshots/packageInfo.json new file mode 100644 index 0000000000..a1ed3e2533 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/packageInfo.json @@ -0,0 +1,16 @@ +{ + "default": { + "name": "snapshot-util", + "version": "0.1" + }, + "special": { + "odd_pkg_name_1": { + "name": "package with space", + "version": "0.1" + }, + "empty_project_name_version": { + "name": "", + "version": "" + } + } +} From 73900cf0b3a63ed9806e2314fc7042ba0f2e13ae Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sat, 17 Oct 2026 21:48:01 -0400 Subject: [PATCH 03/21] symbol table and definitions --- pyrefly/lib/report/glean/convert.rs | 4 +- pyrefly/lib/report/scip.rs | 61 ++- .../scip/snapshots/input/graph_1278/mwe.py | 21 + .../output/aliased_import/aliased.py | 1 + .../output/class_nohint/class_nohint.py | 8 + .../snapshots/output/comprehensions/comp.py | 14 + .../output/empty_project_name_version/main.py | 1 + .../output/enclosing_range_class/advanced.py | 12 + .../output/enclosing_range_class/simple.py | 1 + .../enclosing_range_function/advanced.py | 8 + .../output/enclosing_range_function/simple.py | 2 + .../scip/snapshots/output/f_string/fstring.py | 1 + .../xyz/nested_file.py | 1 + .../scip/snapshots/output/graph_1278/mwe.py | 35 ++ .../nested_items/src/foo/bar/__init__.py | 2 + .../nested_items/src/foo/bar/baz/mod.py | 4 + .../snapshots/output/odd_pkg_name_1/main.py | 1 + .../scip/snapshots/output/pyproject_1/main.py | 1 + .../snapshots/output/relative_import/foo.py | 7 + .../output/single_class/src/single_class.py | 10 + .../single_function/src/single_function.py | 8 + .../output/unique/field_docstring.py | 2 + .../output/unique/grandparent_impl.py | 7 + .../output/unique/inferred_field_docstring.py | 5 + .../snapshots/output/unique/inherits_class.py | 15 + .../output/unique/module_docstring.py | 1 + .../output/unique/multiinherits_test.py | 19 + .../output/unique/property_access.py | 10 + .../output/unique/vars_inside_scopes.py | 7 + .../scip/snapshots/output/unique/walrus.py | 11 + pyrefly/lib/report/scip/symbols.rs | 493 ++++++++++++++++++ 31 files changed, 760 insertions(+), 13 deletions(-) create mode 100644 pyrefly/lib/report/scip/snapshots/input/graph_1278/mwe.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py create mode 100644 pyrefly/lib/report/scip/symbols.rs diff --git a/pyrefly/lib/report/glean/convert.rs b/pyrefly/lib/report/glean/convert.rs index 07a7c575c0..418b32b240 100644 --- a/pyrefly/lib/report/glean/convert.rs +++ b/pyrefly/lib/report/glean/convert.rs @@ -165,7 +165,9 @@ fn file_fact(module_info: &ModuleInfo) -> src::File { src::File::new(relative_path.replace('\\', "/")) } -fn gather_nonlocal_variables(body: &[Stmt]) -> (Arc>, Arc>) { +pub(crate) fn gather_nonlocal_variables( + body: &[Stmt], +) -> (Arc>, Arc>) { let mut globals = SmallSet::new(); let mut nonlocals = SmallSet::new(); for stmt in body { diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 0131dc50a0..89c57226c4 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -9,9 +9,13 @@ use std::path::Path; use lsp_types::Uri; use pyrefly_build::handle::Handle; +use pyrefly_python::module::Module; use pyrefly_python::module_name::ModuleName; use pyrefly_util::absolutize::Absolutize; +use pyrefly_util::lined_buffer::DisplayPos; +use pyrefly_util::lined_buffer::DisplayRange; use pyrefly_util::unix_path::path_to_unix_string; +use ruff_text_size::TextRange; use scip::symbol::format_symbol; use scip::types::Descriptor; use scip::types::Document; @@ -25,10 +29,14 @@ use scip::types::TextEncoding; use scip::types::ToolInfo; use scip::types::descriptor::Suffix; +use crate::report::scip::symbols::ScipSymbol; +use crate::report::scip::symbols::definitions; +use crate::report::scip::symbols::descriptor; use crate::state::state::Transaction; #[cfg(test)] mod snapshot; +mod symbols; const SCHEME: &str = "scip-python"; const MANAGER: &str = "python"; @@ -80,25 +88,53 @@ fn document( ) -> Option { let module = transaction.get_module_info(handle)?; let relative_path = module.path().as_path().strip_prefix(project_root).ok()?; + let ast = transaction.get_ast(handle)?; + let mut occurrences = vec![Occurrence { + range: vec![0, 0, 0], + symbol: module_symbol(project, module.name()), + symbol_roles: SymbolRole::Definition as i32, + ..Default::default() + }]; + occurrences.extend( + definitions(module.name(), &ast) + .into_iter() + .map(|(range, symbol)| Occurrence { + range: scip_range(&module, range), + symbol: format_scip_symbol(project, &symbol), + symbol_roles: SymbolRole::Definition as i32, + ..Default::default() + }), + ); Some(Document { language: "python".to_owned(), relative_path: path_to_unix_string(relative_path), - position_encoding: PositionEncoding::UTF8CodeUnitOffsetFromLineStart.into(), - occurrences: vec![Occurrence { - range: vec![0, 0, 0], - symbol: module_symbol(project, module.name()), - symbol_roles: SymbolRole::Definition as i32, - ..Default::default() - }], + position_encoding: PositionEncoding::UTF32CodeUnitOffsetFromLineStart.into(), + occurrences, ..Default::default() }) } -fn descriptor(name: &str, suffix: Suffix) -> Descriptor { - Descriptor { - name: name.to_owned(), - suffix: suffix.into(), - ..Default::default() +/// A SCIP range: `[line, start, end]` on one line, or `[start line, start, end line, end]`. +fn scip_range(module: &Module, range: TextRange) -> Vec { + let DisplayRange { start, end } = module.display_range(range); + let position = |pos: DisplayPos| { + ( + pos.line_within_file().to_zero_indexed() as i32, + pos.column().get() as i32 - 1, + ) + }; + let ((start_line, start_column), (end_line, end_column)) = (position(start), position(end)); + if start_line == end_line { + vec![start_line, start_column, end_column] + } else { + vec![start_line, start_column, end_line, end_column] + } +} + +fn format_scip_symbol(package: &Package, symbol: &ScipSymbol) -> String { + match symbol { + ScipSymbol::Global(descriptors) => global_symbol(package, descriptors.clone()), + ScipSymbol::Local(id) => format!("local {id}"), } } @@ -235,6 +271,7 @@ mod tests { enclosing_range_function, f_string, file_from_module_import, + graph_1278, nameparts_import, nested_items, odd_pkg_name_1, diff --git a/pyrefly/lib/report/scip/snapshots/input/graph_1278/mwe.py b/pyrefly/lib/report/scip/snapshots/input/graph_1278/mwe.py new file mode 100644 index 0000000000..bdafcec6bb --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/graph_1278/mwe.py @@ -0,0 +1,21 @@ +from typing import TypeVar, Generic, Callable, Iterator, ParamSpec + +_T_co = TypeVar("_T_co") +_P = ParamSpec("_P") + +class X(Generic[_T_co]): + pass + +def decorate(func: Callable[_P, Iterator[_T_co]]) -> Callable[_P, X[_T_co]]: ... + +class Foo: + @decorate + def foo(self) -> Iterator[None]: ... + +@decorate +def noop(): + yield + +class FooImpl(Foo): + def foo(self): + return noop() \ No newline at end of file diff --git a/pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py b/pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py index 853108f780..4335bc015f 100644 --- a/pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py +++ b/pyrefly/lib/report/scip/snapshots/output/aliased_import/aliased.py @@ -1,4 +1,5 @@ # < definition scip-python python snapshot-util 0.1 aliased/__init__: SOME_CONSTANT = "Some Constant" +#^^^^^^^^^^^^ definition snapshot-util 0.1 aliased/SOME_CONSTANT. diff --git a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py index a705912479..e2a576a334 100644 --- a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py +++ b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py @@ -3,15 +3,23 @@ # format-options: showDocs class Example: +# ^^^^^^^ definition snapshot-util 0.1 class_nohint/Example# # Note, only y has a type hint y: int +# ^ definition snapshot-util 0.1 class_nohint/Example#y. def __init__(self, in_val): +# ^^^^^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__(). +# ^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__().(self) +# ^^^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__().(in_val) self.x = in_val +# ^ definition snapshot-util 0.1 class_nohint/Example#x. self.x = self.x + 1 self.y = in_val def something(self): +# ^^^^^^^^^ definition snapshot-util 0.1 class_nohint/Example#something(). +# ^^^^ definition snapshot-util 0.1 class_nohint/Example#something().(self) print(self.x) print(self.y) diff --git a/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py index 6b7a119fde..c4d692baae 100644 --- a/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py +++ b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py @@ -1,16 +1,30 @@ # < definition scip-python python snapshot-util 0.1 comp/__init__: def something(x): +# ^^^^^^^^^ definition snapshot-util 0.1 comp/something(). +# ^ definition snapshot-util 0.1 comp/something().(x) return 2 * x _ = [x for x in "should be local 0"] +# definition snapshot-util 0.1 comp/_. +# ^ definition local 0 _ = [something(x) for x in "should be local 1"] +# definition snapshot-util 0.1 comp/_. +# ^ definition local 1 _ = [something(x) for x in "should be local 2" if x == "s"] +# definition snapshot-util 0.1 comp/_. +# ^ definition local 2 _ = {k: x for (k, x) in enumerate(range(10))} +# definition snapshot-util 0.1 comp/_. +# ^ definition local 3 +# ^ definition local 4 asdf = (var for var in [1, 2, 3] if var % 2 == 0) +#^^^ definition snapshot-util 0.1 comp/asdf. +# ^^^ definition local 5 for var in asdf: +# ^^^ definition snapshot-util 0.1 comp/var. print(var) # TODO: ListComprehensions with if diff --git a/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py b/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py index fd7668a411..fd7495a896 100644 --- a/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py @@ -1,6 +1,7 @@ # < definition scip-python python . . main/__init__: def main(): +# ^^^^ definition . . main/main(). pass main() diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py index 21afdceee3..151160c7ad 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py @@ -2,15 +2,27 @@ # format-options: showRanges def class_decorator(cls): +# ^^^^^^^^^^^^^^^ definition snapshot-util 0.1 advanced/class_decorator(). +# ^^^ definition snapshot-util 0.1 advanced/class_decorator().(cls) def wrapper(*args, **kwargs): +# ^^^^^^^ definition snapshot-util 0.1 advanced/class_decorator().wrapper(). +# ^^^^ definition snapshot-util 0.1 advanced/class_decorator().wrapper().(args) +# ^^^^^^ definition snapshot-util 0.1 advanced/class_decorator().wrapper().(kwargs) return cls(*args, **kwargs) return wrapper @class_decorator class Test: +# ^^^^ definition snapshot-util 0.1 advanced/Test# def __init__(self, x: float): +# ^^^^^^^^ definition snapshot-util 0.1 advanced/Test#__init__(). +# ^^^^ definition snapshot-util 0.1 advanced/Test#__init__().(self) +# ^ definition snapshot-util 0.1 advanced/Test#__init__().(x) self.x = x +# ^ definition snapshot-util 0.1 advanced/Test#x. def test(self) -> float: +# ^^^^ definition snapshot-util 0.1 advanced/Test#test(). +# ^^^^ definition snapshot-util 0.1 advanced/Test#test().(self) return self.x diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py index 65703beeeb..c9c70ea6fd 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py @@ -2,5 +2,6 @@ # format-options: showRanges class Test(): +# ^^^^ definition snapshot-util 0.1 simple/Test# pass diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py index 4d108f48f5..1c7632d25d 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py @@ -2,13 +2,21 @@ # format-options: showRanges def decorator(func): +# ^^^^^^^^^ definition snapshot-util 0.1 advanced/decorator(). +# ^^^^ definition snapshot-util 0.1 advanced/decorator().(func) def wrapper(*args, **kwargs): +# ^^^^^^^ definition snapshot-util 0.1 advanced/decorator().wrapper(). +# ^^^^ definition snapshot-util 0.1 advanced/decorator().wrapper().(args) +# ^^^^^^ definition snapshot-util 0.1 advanced/decorator().wrapper().(kwargs) return func(*args, **kwargs) return wrapper @decorator def func(x: float) -> float: +# ^^^^ definition snapshot-util 0.1 advanced/func(). +# ^ definition snapshot-util 0.1 advanced/func().(x) test = x +# ^^^^ definition local 0 return test diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py index aef0aa63ff..37e9f8e6b0 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py @@ -2,5 +2,7 @@ # format-options: showRanges def simple(x): +# ^^^^^^ definition snapshot-util 0.1 simple/simple(). +# ^ definition snapshot-util 0.1 simple/simple().(x) return x diff --git a/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py b/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py index 3a5a355b8e..88ec7923b3 100644 --- a/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py @@ -1,6 +1,7 @@ # < definition scip-python python snapshot-util 0.1 fstring/__init__: var = ", world!" +#^^ definition snapshot-util 0.1 fstring/var. print(f"var: hello {var}") diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py index 3beac87099..62f17e5b7f 100644 --- a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/xyz/nested_file.py @@ -1,4 +1,5 @@ # < definition scip-python python snapshot-util 0.1 `xyz.nested_file`/__init__: X = "a constant" +# definition snapshot-util 0.1 `xyz.nested_file`/X. diff --git a/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py new file mode 100644 index 0000000000..a4b697558f --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py @@ -0,0 +1,35 @@ +# < definition scip-python python snapshot-util 0.1 mwe/__init__: + +from typing import TypeVar, Generic, Callable, Iterator, ParamSpec + +_T_co = TypeVar("_T_co") +#^^^^ definition snapshot-util 0.1 mwe/_T_co. +_P = ParamSpec("_P") +#^ definition snapshot-util 0.1 mwe/_P. + +class X(Generic[_T_co]): +# ^ definition snapshot-util 0.1 mwe/X# + pass + +def decorate(func: Callable[_P, Iterator[_T_co]]) -> Callable[_P, X[_T_co]]: ... +# ^^^^^^^^ definition snapshot-util 0.1 mwe/decorate(). +# ^^^^ definition snapshot-util 0.1 mwe/decorate().(func) + +class Foo: +# ^^^ definition snapshot-util 0.1 mwe/Foo# + @decorate + def foo(self) -> Iterator[None]: ... +# ^^^ definition snapshot-util 0.1 mwe/Foo#foo(). +# ^^^^ definition snapshot-util 0.1 mwe/Foo#foo().(self) + +@decorate +def noop(): +# ^^^^ definition snapshot-util 0.1 mwe/noop(). + yield + +class FooImpl(Foo): +# ^^^^^^^ definition snapshot-util 0.1 mwe/FooImpl# + def foo(self): +# ^^^ definition snapshot-util 0.1 mwe/FooImpl#foo(). +# ^^^^ definition snapshot-util 0.1 mwe/FooImpl#foo().(self) + return noop() diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py index 2b85a27b40..29a739c24e 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py @@ -1,5 +1,7 @@ # < definition scip-python python snapshot-util 0.1 `src.foo.bar`/__init__: class InitClass: +# ^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar`/InitClass# init_item = 10 +# ^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar`/InitClass#init_item. diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py index 74f0bab410..460d346aa6 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py @@ -1,8 +1,12 @@ # < definition scip-python python snapshot-util 0.1 `src.foo.bar.baz.mod`/__init__: class SuchNestedMuchWow: +# ^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow# class_item: int = 42 +# ^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow#class_item. class AnotherNestedMuchWow: +# ^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow# other_item: int = 42 +# ^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow#other_item. diff --git a/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py b/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py index 85ba6304a5..32d5f03bf3 100644 --- a/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py @@ -1,6 +1,7 @@ # < definition scip-python python package with space 0.1 main/__init__: def main(): +# ^^^^ definition package with space 0.1 main/main(). pass main() diff --git a/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py index 2a36ab316e..2e04f3e619 100644 --- a/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py @@ -1,6 +1,7 @@ # < definition scip-python python snapshot-util 0.1 main/__init__: def main(): +# ^^^^ definition snapshot-util 0.1 main/main(). pass main() diff --git a/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py b/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py index 4023b5e63e..e2a2bd044a 100644 --- a/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py +++ b/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py @@ -1,16 +1,23 @@ # < definition scip-python python snapshot-util 0.1 foo/__init__: def exported_function(): +# ^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 foo/exported_function(). return "function" class MyClass: +# ^^^^^^^ definition snapshot-util 0.1 foo/MyClass# """This is a class and it is cool""" def __init__(self): +# ^^^^^^^^ definition snapshot-util 0.1 foo/MyClass#__init__(). +# ^^^^ definition snapshot-util 0.1 foo/MyClass#__init__().(self) pass def exported_function(self): +# ^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 foo/MyClass#exported_function(). +# ^^^^ definition snapshot-util 0.1 foo/MyClass#exported_function().(self) return "exported" this_class = MyClass() +#^^^^^^^^^ definition snapshot-util 0.1 foo/this_class. diff --git a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py index e339e36518..0a475374ba 100644 --- a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py @@ -1,14 +1,24 @@ # < definition scip-python python snapshot-util 0.1 `src.single_class`/__init__: class ExampleClass: +# ^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass# a: int +# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#a. b: int +# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#b. c: str +# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#c. static_var = "Hello World" +# ^^^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#static_var. def __init__(self, a: int, b: int): +# ^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__(). +# ^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) +# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(a) +# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(b) local_c = ", world!" +# ^^^^^^^ definition local 0 self.a = a self.b = b diff --git a/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py index 372bb931fb..d6b65e1f55 100644 --- a/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py +++ b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py @@ -1,14 +1,22 @@ # < definition scip-python python snapshot-util 0.1 `src.single_function`/__init__: def my_cool_function(a: str) -> str: +# ^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/my_cool_function(). +# ^ definition snapshot-util 0.1 `src.single_function`/my_cool_function().(a) x = ", world" +# ^ definition local 0 return a + x def my_cool_function_2(a: str): +# ^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/my_cool_function_2(). +# ^ definition snapshot-util 0.1 `src.single_function`/my_cool_function_2().(a) x = ", world" +# ^ definition local 1 return (lambda y: a + x + y)("oh no") +# ^ definition local 2 def next_level(): +# ^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/next_level(). return my_cool_function my_cool_function("hello") diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py index 0137516f42..0db4f3d978 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py @@ -3,7 +3,9 @@ # format-options: showDocs class ClassWithField: +# ^^^^^^^^^^^^^^ definition snapshot-util 0.1 field_docstring/ClassWithField# a: int +# ^ definition snapshot-util 0.1 field_docstring/ClassWithField#a. """Hello world, this is a""" diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py index d35c1803ba..d1a56b2712 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py @@ -1,13 +1,20 @@ # < definition scip-python python snapshot-util 0.1 grandparent_impl/__init__: class A: +# ^ definition snapshot-util 0.1 grandparent_impl/A# def grandparent(self) -> bool: +# ^^^^^^^^^^^ definition snapshot-util 0.1 grandparent_impl/A#grandparent(). +# ^^^^ definition snapshot-util 0.1 grandparent_impl/A#grandparent().(self) return True class B(A): +# ^ definition snapshot-util 0.1 grandparent_impl/B# ... class C(B): +# ^ definition snapshot-util 0.1 grandparent_impl/C# def grandparent(self) -> bool: +# ^^^^^^^^^^^ definition snapshot-util 0.1 grandparent_impl/C#grandparent(). +# ^^^^ definition snapshot-util 0.1 grandparent_impl/C#grandparent().(self) return False diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py index 574700506f..c11f2db9db 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py @@ -3,6 +3,11 @@ # format-options: showDocs class ClassWithInferredField: +# ^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField# def __init__(self, b: int): +# ^^^^^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__(). +# ^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(self) +# ^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(b) self.b = b +# ^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#b. diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py index d0e019f823..d3f6ba18db 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py @@ -1,19 +1,34 @@ # < definition scip-python python snapshot-util 0.1 inherits_class/__init__: class A: +# ^ definition snapshot-util 0.1 inherits_class/A# def x(self) -> int: +# ^ definition snapshot-util 0.1 inherits_class/A#x(). +# ^^^^ definition snapshot-util 0.1 inherits_class/A#x().(self) raise NotImplemented def matched_despite_different_type(self, x: int): +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inherits_class/A#matched_despite_different_type(). +# ^^^^ definition snapshot-util 0.1 inherits_class/A#matched_despite_different_type().(self) +# ^ definition snapshot-util 0.1 inherits_class/A#matched_despite_different_type().(x) pass class B(A): +# ^ definition snapshot-util 0.1 inherits_class/B# def x(self) -> int: +# ^ definition snapshot-util 0.1 inherits_class/B#x(). +# ^^^^ definition snapshot-util 0.1 inherits_class/B#x().(self) return 5 def matched_despite_different_type(self, x: int, y: int): +# ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type(). +# ^^^^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(self) +# ^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(x) +# ^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(y) pass def unrelated(self): +# ^^^^^^^^^ definition snapshot-util 0.1 inherits_class/B#unrelated(). +# ^^^^ definition snapshot-util 0.1 inherits_class/B#unrelated().(self) pass diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py index 5ea528b192..68ac39fa05 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py @@ -7,5 +7,6 @@ """ def x(): +# ^ definition snapshot-util 0.1 module_docstring/x(). return 1 diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py index 3f890f4713..a31e819920 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py @@ -1,29 +1,48 @@ # < definition scip-python python snapshot-util 0.1 multiinherits_test/__init__: class Left: +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Left# def one(self) -> int: +# ^^^ definition snapshot-util 0.1 multiinherits_test/Left#one(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Left#one().(self) return 1 def shared(self) -> bool: +# ^^^^^^ definition snapshot-util 0.1 multiinherits_test/Left#shared(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Left#shared().(self) return False class Right: +# ^^^^^ definition snapshot-util 0.1 multiinherits_test/Right# def two(self): +# ^^^ definition snapshot-util 0.1 multiinherits_test/Right#two(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Right#two().(self) return 2 def shared(self) -> bool: +# ^^^^^^ definition snapshot-util 0.1 multiinherits_test/Right#shared(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Right#shared().(self) return False class Multi(Left, Right): +# ^^^^^ definition snapshot-util 0.1 multiinherits_test/Multi# def one(self) -> int: +# ^^^ definition snapshot-util 0.1 multiinherits_test/Multi#one(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#one().(self) return 1 def two(self): +# ^^^ definition snapshot-util 0.1 multiinherits_test/Multi#two(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#two().(self) return 2 def three(self): +# ^^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#three(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#three().(self) return 3 def shared(self) -> bool: +# ^^^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#shared(). +# ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#shared().(self) return True diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py index f8395bb251..60b24e7200 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py @@ -3,18 +3,28 @@ from typing import Sequence class PropertyClass: +# ^^^^^^^^^^^^^ definition snapshot-util 0.1 property_access/PropertyClass# def __init__(self): +# ^^^^^^^^ definition snapshot-util 0.1 property_access/PropertyClass#__init__(). +# ^^^^ definition snapshot-util 0.1 property_access/PropertyClass#__init__().(self) pass @property def prop_ref(self): +# ^^^^^^^^ definition snapshot-util 0.1 property_access/PropertyClass#prop_ref(). +# ^^^^ definition snapshot-util 0.1 property_access/PropertyClass#prop_ref().(self) return 5 xs = [PropertyClass()] +#^ definition snapshot-util 0.1 property_access/xs. def usage(xs: Sequence[PropertyClass]): +# ^^^^^ definition snapshot-util 0.1 property_access/usage(). +# ^^ definition snapshot-util 0.1 property_access/usage().(xs) def nested(): +# ^^^^^^ definition snapshot-util 0.1 property_access/usage().nested(). for x in xs: +# ^ definition local 0 print(x.prop_ref) diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py index df2d0be2f3..029b95563c 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py @@ -3,12 +3,19 @@ from typing import List class X: +# ^ definition snapshot-util 0.1 vars_inside_scopes/X# items: List[int] +# ^^^^^ definition snapshot-util 0.1 vars_inside_scopes/X#items. def my_func(self): +# ^^^^^^^ definition snapshot-util 0.1 vars_inside_scopes/X#my_func(). +# ^^^^ definition snapshot-util 0.1 vars_inside_scopes/X#my_func().(self) for x in self.items: +# ^ definition local 0 y = x + 1 +# ^ definition local 1 if 5 in self.items: z = "oh ya" +# ^ definition local 2 diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py index 1f01659f95..5654899dbf 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py @@ -3,28 +3,39 @@ import re some_list = [1, 2, 3, 4, 5, 100, 1000] +#^^^^^^^^ definition snapshot-util 0.1 walrus/some_list. pattern = "^[A-Za-z0-9_]*$" +#^^^^^^ definition snapshot-util 0.1 walrus/pattern. text = "Some_name123" +#^^^ definition snapshot-util 0.1 walrus/text. # if statement if (n := len(some_list)) > 10: +# ^ definition snapshot-util 0.1 walrus/n. print(f"List is too long with {n} elements.") if (match := re.search(pattern, text)) is not None: +# ^^^^^ definition snapshot-util 0.1 walrus/match. print("Found match:", match.group(0)) # comprehensions def show_some_comprehension(): +# ^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 walrus/show_some_comprehension(). [print(x, root) for x in some_list if (x % 2 == 0) and (root := x**0.5) > 5] +# ^ definition local 0 +# ^^^^ definition local 1 # while loop while (line := input("Enter text: ")) != "quit": +# ^^^^ definition snapshot-util 0.1 walrus/line. print(f"You entered: {line}") # if + comprehension if any((any_n := num) < 0 for num in some_list): +# ^^^^^ definition snapshot-util 0.1 walrus/any_n. +# ^^^ definition local 2 print(f"Negative number found: {any_n}") diff --git a/pyrefly/lib/report/scip/symbols.rs b/pyrefly/lib/report/scip/symbols.rs new file mode 100644 index 0000000000..b9002fb230 --- /dev/null +++ b/pyrefly/lib/report/scip/symbols.rs @@ -0,0 +1,493 @@ +/* + * Copyright (c) Meta Platforms, Inc. and affiliates. + * + * This source code is licensed under the MIT license found in the + * LICENSE file in the root directory of this source tree. + */ + +use std::collections::HashMap; + +use pyrefly_python::ast::Ast; +use pyrefly_python::module_name::ModuleName; +use ruff_python_ast::Comprehension; +use ruff_python_ast::ExceptHandler; +use ruff_python_ast::Expr; +use ruff_python_ast::ExprAttribute; +use ruff_python_ast::ExprContext; +use ruff_python_ast::Identifier; +use ruff_python_ast::ModModule; +use ruff_python_ast::Parameters; +use ruff_python_ast::Pattern; +use ruff_python_ast::Stmt; +use ruff_python_ast::name::Name; +use ruff_python_ast::visitor::source_order::SourceOrderVisitor; +use ruff_python_ast::visitor::source_order::walk_except_handler; +use ruff_python_ast::visitor::source_order::walk_expr; +use ruff_python_ast::visitor::source_order::walk_pattern; +use ruff_python_ast::visitor::source_order::walk_stmt; +use ruff_text_size::TextRange; +use scip::types::Descriptor; +use scip::types::descriptor::Suffix; +use starlark_map::small_set::SmallSet; + +use crate::report::glean::convert::gather_nonlocal_variables; + +#[derive(Debug, Clone, PartialEq)] +pub enum ScipSymbol { + Global(Vec), + Local(usize), +} + +/// The SCIP symbol of every name a module defines, keyed by the defining identifier, in source +/// order. +pub fn definitions(module: ModuleName, ast: &ModModule) -> Vec<(TextRange, ScipSymbol)> { + let module_path = vec![descriptor(module.as_str(), Suffix::Namespace)]; + let mut builder = Builder { + scopes: vec![Scope::new(ScopeKind::Module, module_path.clone())], + module_path, + definitions: Vec::new(), + next_local: 0, + }; + builder.visit_body(&ast.body); + builder.definitions +} + +pub fn descriptor(name: &str, suffix: Suffix) -> Descriptor { + Descriptor { + name: name.to_owned(), + suffix: suffix.into(), + ..Default::default() + } +} + +enum ScopeKind { + Module, + Class { + /// Names bound directly in the class body, plus `self.x` attributes seen so far. + attributes: SmallSet, + }, + Function { + receiver: Option, + globals: SmallSet, + nonlocals: SmallSet, + }, + /// A comprehension or lambda: every name it binds is a local. + Anonymous, +} + +struct Scope { + kind: ScopeKind, + path: Vec, + names: HashMap, +} + +impl Scope { + fn new(kind: ScopeKind, path: Vec) -> Self { + Self { + kind, + path, + names: HashMap::new(), + } + } +} + +struct Builder { + module_path: Vec, + scopes: Vec, + definitions: Vec<(TextRange, ScipSymbol)>, + next_local: usize, +} + +fn child(path: &[Descriptor], name: &str, suffix: Suffix) -> Vec { + let mut path = path.to_vec(); + path.push(descriptor(name, suffix)); + path +} + +impl Builder { + fn scope(&self) -> &Scope { + self.scopes + .last() + .expect("the module scope is never popped") + } + + fn new_local(&mut self) -> ScipSymbol { + self.next_local += 1; + ScipSymbol::Local(self.next_local - 1) + } + + fn define(&mut self, id: &Identifier, symbol: ScipSymbol) { + self.definitions.push((id.range, symbol)); + } + + /// Define a name in the current scope, regardless of any earlier binding of it. + fn define_in_scope(&mut self, id: &Identifier, symbol: ScipSymbol) { + self.scopes + .last_mut() + .expect("the module scope is never popped") + .names + .insert(id.id.clone(), symbol.clone()); + self.define(id, symbol); + } + + /// Define a function or class in the current scope, returning its path. + fn declare(&mut self, id: &Identifier, suffix: Suffix) -> Vec { + let path = child(&self.scope().path, &id.id, suffix); + self.define_in_scope(id, ScipSymbol::Global(path.clone())); + path + } + + /// The symbol for binding `name` in scope `index`, creating it on first use. + fn binding(&mut self, index: usize, name: &Name) -> ScipSymbol { + if let Some(symbol) = self.scopes[index].names.get(name) { + return symbol.clone(); + } + let symbol = match &self.scopes[index].kind { + ScopeKind::Module | ScopeKind::Class { .. } => { + ScipSymbol::Global(child(&self.scopes[index].path, name, Suffix::Term)) + } + ScopeKind::Function { globals, .. } if globals.contains(name) => { + ScipSymbol::Global(child(&self.module_path, name, Suffix::Term)) + } + ScopeKind::Function { nonlocals, .. } if nonlocals.contains(name) => { + match (0..index) + .rev() + .find(|i| matches!(self.scopes[*i].kind, ScopeKind::Function { .. })) + { + Some(enclosing) => self.binding(enclosing, name), + None => self.new_local(), + } + } + ScopeKind::Function { .. } | ScopeKind::Anonymous => self.new_local(), + }; + self.scopes[index] + .names + .insert(name.clone(), symbol.clone()); + symbol + } + + /// Bind a name in the current scope. A walrus inside a comprehension binds in the enclosing + /// scope instead. + fn bind(&mut self, id: &Identifier, walrus: bool) { + let mut index = self.scopes.len() - 1; + if walrus { + while index > 0 && matches!(self.scopes[index].kind, ScopeKind::Anonymous) { + index -= 1; + } + } + let symbol = self.binding(index, &id.id); + self.define(id, symbol); + } + + /// `self.x = ...` in a method defines the attribute `x`, unless the class already did. + fn bind_attribute(&mut self, attribute: &ExprAttribute) { + let [.., class, method] = self.scopes.as_mut_slice() else { + return; + }; + let ( + ScopeKind::Class { attributes }, + ScopeKind::Function { + receiver: Some(receiver), + .. + }, + ) = (&mut class.kind, &method.kind) + else { + return; + }; + if let Some(attr) = Ast::expr_receiver_attr(attribute, receiver) + && attributes.insert(attr.id.clone()) + { + let symbol = ScipSymbol::Global(child(&class.path, &attr.id, Suffix::Term)); + self.define(attr, symbol); + } + } + + fn bind_parameters(&mut self, parameters: &Parameters) { + for parameter in parameters.iter() { + let id = parameter.name(); + let symbol = match &self.scope().kind { + ScopeKind::Anonymous => self.new_local(), + _ => ScipSymbol::Global(child(&self.scope().path, &id.id, Suffix::Parameter)), + }; + self.define_in_scope(id, symbol); + } + } + + fn visit_comprehensions(&mut self, generators: &[Comprehension], elements: &[&Expr]) { + // The first iterable is evaluated in the enclosing scope. + if let Some(first) = generators.first() { + self.visit_expr(&first.iter); + } + self.scopes + .push(Scope::new(ScopeKind::Anonymous, self.scope().path.clone())); + for (i, generator) in generators.iter().enumerate() { + if i > 0 { + self.visit_expr(&generator.iter); + } + self.visit_expr(&generator.target); + for condition in &generator.ifs { + self.visit_expr(condition); + } + } + for element in elements { + self.visit_expr(element); + } + self.scopes.pop(); + } +} + +impl<'a> SourceOrderVisitor<'a> for Builder { + fn visit_stmt(&mut self, stmt: &'a Stmt) { + match stmt { + Stmt::FunctionDef(function) => { + for decorator in &function.decorator_list { + self.visit_decorator(decorator); + } + self.visit_parameters(&function.parameters); + if let Some(returns) = &function.returns { + self.visit_expr(returns); + } + let path = self.declare(&function.name, Suffix::Method); + let receiver = match &self.scope().kind { + ScopeKind::Class { .. } => function + .parameters + .iter_non_variadic_params() + .next() + .map(|p| p.parameter.name.id.clone()), + _ => None, + }; + let (globals, nonlocals) = gather_nonlocal_variables(&function.body); + self.scopes.push(Scope::new( + ScopeKind::Function { + receiver, + globals: (*globals).clone(), + nonlocals: (*nonlocals).clone(), + }, + path, + )); + self.bind_parameters(&function.parameters); + self.visit_body(&function.body); + self.scopes.pop(); + } + Stmt::ClassDef(class) => { + for decorator in &class.decorator_list { + self.visit_decorator(decorator); + } + if let Some(arguments) = &class.arguments { + self.visit_arguments(arguments); + } + let path = self.declare(&class.name, Suffix::Type); + let mut attributes = SmallSet::new(); + for stmt in &class.body { + match stmt { + Stmt::FunctionDef(x) => { + attributes.insert(x.name.id.clone()); + } + Stmt::ClassDef(x) => { + attributes.insert(x.name.id.clone()); + } + Stmt::Assign(x) => x.targets.iter().for_each(|t| { + Ast::expr_lvalue(t, &mut |n| { + attributes.insert(n.id.clone()); + }) + }), + Stmt::AnnAssign(x) => Ast::expr_lvalue(&x.target, &mut |n| { + attributes.insert(n.id.clone()); + }), + _ => {} + } + } + self.scopes + .push(Scope::new(ScopeKind::Class { attributes }, path)); + self.visit_body(&class.body); + self.scopes.pop(); + } + Stmt::AugAssign(assign) => { + if !matches!(&*assign.target, Expr::Name(_)) { + self.visit_expr(&assign.target); + } + self.visit_expr(&assign.value); + } + _ => walk_stmt(self, stmt), + } + } + + fn visit_expr(&mut self, expr: &'a Expr) { + match expr { + Expr::Name(name) if name.ctx == ExprContext::Store => { + self.bind(&Ast::expr_name_identifier(name.clone()), false) + } + Expr::Named(named) => { + if let Expr::Name(name) = &*named.target { + self.bind(&Ast::expr_name_identifier(name.clone()), true); + } + self.visit_expr(&named.value); + } + Expr::Attribute(attribute) if attribute.ctx == ExprContext::Store => { + self.bind_attribute(attribute); + self.visit_expr(&attribute.value); + } + Expr::ListComp(x) => self.visit_comprehensions(&x.generators, &[&x.elt]), + Expr::SetComp(x) => self.visit_comprehensions(&x.generators, &[&x.elt]), + Expr::Generator(x) => self.visit_comprehensions(&x.generators, &[&x.elt]), + Expr::DictComp(x) => { + let elements: Vec<&Expr> = + x.key.as_deref().into_iter().chain([&*x.value]).collect(); + self.visit_comprehensions(&x.generators, &elements) + } + Expr::Lambda(lambda) => { + if let Some(parameters) = &lambda.parameters { + self.visit_parameters(parameters); + } + self.scopes + .push(Scope::new(ScopeKind::Anonymous, self.scope().path.clone())); + if let Some(parameters) = &lambda.parameters { + self.bind_parameters(parameters); + } + self.visit_expr(&lambda.body); + self.scopes.pop(); + } + _ => walk_expr(self, expr), + } + } + + fn visit_except_handler(&mut self, handler: &'a ExceptHandler) { + let ExceptHandler::ExceptHandler(x) = handler; + if let Some(name) = &x.name { + self.bind(name, false); + } + walk_except_handler(self, handler); + } + + fn visit_pattern(&mut self, pattern: &'a Pattern) { + let name = match pattern { + Pattern::MatchAs(x) => x.name.as_ref(), + Pattern::MatchStar(x) => x.name.as_ref(), + Pattern::MatchMapping(x) => x.rest.as_ref(), + _ => None, + }; + if let Some(name) = name { + self.bind(name, false); + } + walk_pattern(self, pattern); + } +} + +#[cfg(test)] +mod tests { + use pyrefly_python::ast::Ast; + use ruff_python_ast::PySourceType; + use scip::symbol::format_symbol; + use scip::types::Symbol; + + use super::*; + + fn symbols(code: &str) -> Vec<(String, String)> { + let (ast, _, _) = Ast::parse(code, PySourceType::Python); + definitions(ModuleName::from_str("m"), &ast) + .iter() + .map(|(range, symbol)| { + let symbol = match symbol { + ScipSymbol::Global(descriptors) => format_symbol(Symbol { + scheme: "s".to_owned(), + descriptors: descriptors.clone(), + ..Default::default() + }), + ScipSymbol::Local(i) => format!("local {i}"), + }; + (code[*range].to_owned(), symbol) + }) + .collect() + } + + fn assert_symbols(code: &str, expected: &[(&str, &str)]) { + let expected: Vec<_> = expected + .iter() + .map(|(name, symbol)| (name.to_string(), format!("s . . . {symbol}"))) + .collect(); + assert_eq!(symbols(code), expected); + } + + #[test] + fn test_classes_and_functions() { + assert_symbols( + r#" +class C: + x: int + class D: ... + def f(self, a, *args, **kwargs): + def g(): ... +"#, + &[ + ("C", "m/C#"), + ("x", "m/C#x."), + ("D", "m/C#D#"), + ("f", "m/C#f()."), + ("self", "m/C#f().(self)"), + ("a", "m/C#f().(a)"), + ("args", "m/C#f().(args)"), + ("kwargs", "m/C#f().(kwargs)"), + ("g", "m/C#f().g()."), + ], + ); + } + + #[test] + fn test_self_attributes() { + assert_symbols( + r#" +class C: + y: int + def __init__(self, v): + self.x = v + self.x = v + self.y = v +"#, + &[ + ("C", "m/C#"), + ("y", "m/C#y."), + ("__init__", "m/C#__init__()."), + ("self", "m/C#__init__().(self)"), + ("v", "m/C#__init__().(v)"), + ("x", "m/C#x."), + ], + ); + } + + #[test] + fn test_locals() { + let actual = symbols( + r#" +x = 1 +for y in []: pass +def f(p): + p = 1 + z = [w for w in []] + z = 2 + global x + x = 3 +squares = [w * w for w in []] +"#, + ); + let expected = [ + ("x", "s . . . m/x."), + ("y", "s . . . m/y."), + ("f", "s . . . m/f()."), + ("p", "s . . . m/f().(p)"), + ("p", "s . . . m/f().(p)"), + ("z", "local 0"), + ("w", "local 1"), + ("z", "local 0"), + ("x", "s . . . m/x."), + ("squares", "s . . . m/squares."), + ("w", "local 2"), + ]; + assert_eq!( + actual, + expected + .iter() + .map(|(a, b)| (a.to_string(), b.to_string())) + .collect::>() + ); + } +} From 26125d4d7e70134885e28d4adcff916806dd8dac Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sat, 17 Oct 2026 22:19:29 -0400 Subject: [PATCH 04/21] handle package name/version from pyproject.toml --- pyrefly/lib/commands/check.rs | 22 ++++++++- pyrefly/lib/report/scip.rs | 49 ++++++++++++++++--- .../scip/snapshots/output/pyproject_1/main.py | 4 +- 3 files changed, 65 insertions(+), 10 deletions(-) diff --git a/pyrefly/lib/commands/check.rs b/pyrefly/lib/commands/check.rs index 10fe708bc2..a6bc62773b 100644 --- a/pyrefly/lib/commands/check.rs +++ b/pyrefly/lib/commands/check.rs @@ -327,6 +327,12 @@ struct OutputArgs { /// Generate a SCIP index for the project #[arg(long, value_name = "OUTPUT_FILE")] report_scip: Option, + /// Package name for the project's symbols in the SCIP index. Defaults to the name in `pyproject.toml`. + #[arg(long, requires = "report_scip")] + report_scip_project_name: Option, + /// Package version for the project's symbols in the SCIP index. Defaults to the version in `pyproject.toml`. + #[arg(long, requires = "report_scip")] + report_scip_project_version: Option, /// Generate a Pysa-compatible JSON file for each module #[arg(long, value_name = "OUTPUT_FILE")] report_pysa: Option, @@ -497,6 +503,8 @@ impl OutputArgs { report_glean, report_glean_ownership, report_scip, + report_scip_project_name, + report_scip_project_version, report_pysa, report_pysa_format, report_demand_tree, @@ -542,6 +550,8 @@ impl OutputArgs { || report_glean.is_some() || *report_glean_ownership || report_scip.is_some() + || report_scip_project_name.is_some() + || report_scip_project_version.is_some() || report_pysa.is_some() || !matches!(report_pysa_format, report::pysa::PysaFormat::Capnp) || report_demand_tree.is_some() @@ -2276,7 +2286,11 @@ impl CheckArgs { transaction, handles, &relative_to, - &report::scip::Package::default(), + &report::scip::Package::project( + &relative_to, + self.output.report_scip_project_name.clone(), + self.output.report_scip_project_version.clone(), + )?, version, ) .write_to_bytes()?, @@ -2581,6 +2595,8 @@ def go(w: Widget) -> int: "check", "--report-scip", scip_path.to_str().unwrap(), + "--report-scip-project-name", + "demo", "--relative-to", root.to_str().unwrap(), ]); @@ -2602,6 +2618,10 @@ def go(w: Widget) -> int: let index = scip::types::Index::parse_from_bytes(&fs::read(&scip_path).unwrap()).unwrap(); assert_eq!(index.documents.len(), 1); assert_eq!(index.documents[0].relative_path, "main.py"); + assert_eq!( + index.documents[0].occurrences[0].symbol, + "scip-python python demo . main/__init__:" + ); let metadata = index.metadata.unwrap(); assert_eq!(metadata.tool_info.name, "pyrefly"); assert_eq!(metadata.tool_info.version, "test"); diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 89c57226c4..261a2b8313 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -12,6 +12,7 @@ use pyrefly_build::handle::Handle; use pyrefly_python::module::Module; use pyrefly_python::module_name::ModuleName; use pyrefly_util::absolutize::Absolutize; +use pyrefly_util::fs_anyhow; use pyrefly_util::lined_buffer::DisplayPos; use pyrefly_util::lined_buffer::DisplayRange; use pyrefly_util::unix_path::path_to_unix_string; @@ -29,6 +30,7 @@ use scip::types::TextEncoding; use scip::types::ToolInfo; use scip::types::descriptor::Suffix; +use crate::config::config::ConfigFile; use crate::report::scip::symbols::ScipSymbol; use crate::report::scip::symbols::definitions; use crate::report::scip::symbols::descriptor; @@ -41,12 +43,42 @@ mod symbols; const SCHEME: &str = "scip-python"; const MANAGER: &str = "python"; -#[derive(Debug, Clone, Default)] +#[derive(Debug, Clone)] pub struct Package { pub name: String, pub version: String, } +impl Package { + /// The package of the project's own symbols. Explicit values come first, then `[project]` + /// and `[tool.poetry]` in the project root's `pyproject.toml`. + pub fn project( + project_root: &Path, + name: Option, + version: Option, + ) -> anyhow::Result { + let pyproject = project_root.join(ConfigFile::PYPROJECT_FILE_NAME); + let pyproject: toml::Table = if (name.is_none() || version.is_none()) && pyproject.exists() + { + fs_anyhow::read_to_string(&pyproject)?.parse()? + } else { + toml::Table::new() + }; + let lookup = |key: &str| { + let poetry = pyproject.get("tool").and_then(|tool| tool.get("poetry")); + [pyproject.get("project"), poetry] + .into_iter() + .flatten() + .find_map(|table| Some(table.get(key)?.as_str()?.to_owned())) + }; + // An empty name or version is written as `.` in symbols. + Ok(Self { + name: name.or_else(|| lookup("name")).unwrap_or_default(), + version: version.or_else(|| lookup("version")).unwrap_or_default(), + }) + } +} + pub fn scip( transaction: &Transaction, handles: &[Handle], @@ -170,7 +202,6 @@ mod tests { use std::path::PathBuf; use pyrefly_python::module_path::ModulePath; - use pyrefly_util::fs_anyhow; use super::*; use crate::report::scip::snapshot::format_snapshot; @@ -183,16 +214,20 @@ mod tests { std::env::current_dir().unwrap().join(path) } + /// A test case uses its own `pyproject.toml` when it has one. fn project_package(snapshots: &Path, case: &str) -> Package { let info: serde_json::Value = serde_json::from_str( &fs_anyhow::read_to_string(&snapshots.join("packageInfo.json")).unwrap(), ) .unwrap(); - let package = info["special"].get(case).unwrap_or(&info["default"]); - Package { - name: package["name"].as_str().unwrap().to_owned(), - version: package["version"].as_str().unwrap().to_owned(), - } + let input_dir = snapshots.join("input").join(case); + let package = match info["special"].get(case) { + Some(package) => Some(package), + None if input_dir.join(ConfigFile::PYPROJECT_FILE_NAME).exists() => None, + None => Some(&info["default"]), + }; + let field = |key: &str| package.map(|package| package[key].as_str().unwrap().to_owned()); + Package::project(&input_dir, field("name"), field("version")).unwrap() } fn assert_snapshot(case: &str) { diff --git a/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py index 2e04f3e619..f404c12282 100644 --- a/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py @@ -1,7 +1,7 @@ -# < definition scip-python python snapshot-util 0.1 main/__init__: +# < definition scip-python python abc 16.05 main/__init__: def main(): -# ^^^^ definition snapshot-util 0.1 main/main(). +# ^^^^ definition abc 16.05 main/main(). pass main() From f2e88b0b365b767b90541c02e3e8cd0170f8d06e Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sun, 18 Oct 2026 00:07:26 -0400 Subject: [PATCH 05/21] name references --- pyrefly/lib/report/scip.rs | 190 ++++++++++++++---- .../snapshots/output/aliased_import/actual.py | 2 + .../output/builtin_imports/builtin_imports.py | 4 + .../output/class_nohint/class_nohint.py | 11 + .../snapshots/output/comprehensions/comp.py | 16 ++ .../snapshots/output/dunder_vars/__main__.py | 2 + .../output/empty_project_name_version/main.py | 1 + .../output/enclosing_range_class/advanced.py | 10 + .../enclosing_range_function/advanced.py | 9 + .../output/enclosing_range_function/simple.py | 1 + .../scip/snapshots/output/f_string/fstring.py | 2 + .../file_from_module_import/abc/file.py | 2 + .../file_from_module.py | 2 + .../scip/snapshots/output/graph_1278/mwe.py | 17 ++ .../output/nameparts_import/nameparts.py | 2 + .../nested_items/src/foo/bar/baz/mod.py | 2 + .../output/nested_items/src/importer.py | 6 + .../output/nested_items/src/long_importer.py | 2 + .../snapshots/output/odd_pkg_name_1/main.py | 1 + .../scip/snapshots/output/pyproject_1/main.py | 1 + .../snapshots/output/relative_import/bar.py | 2 + .../snapshots/output/relative_import/foo.py | 1 + .../output/single_class/src/single_class.py | 11 + .../single_function/src/single_function.py | 11 + .../output/unique/builtin_import_refs.py | 2 + .../output/unique/field_docstring.py | 1 + .../output/unique/grandparent_impl.py | 4 + .../output/unique/inferred_field_docstring.py | 3 + .../snapshots/output/unique/inherits_class.py | 7 + .../output/unique/multiinherits_test.py | 7 + .../output/unique/property_access.py | 7 + .../output/unique/vars_inside_scopes.py | 5 + .../scip/snapshots/output/unique/walrus.py | 23 +++ .../output/unresolved_import/unresolved.py | 4 + 34 files changed, 337 insertions(+), 34 deletions(-) diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 261a2b8313..f8768950f6 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -5,17 +5,30 @@ * LICENSE file in the root directory of this source tree. */ +use std::collections::HashMap; use std::path::Path; +use std::sync::Arc; +use dupe::Dupe; use lsp_types::Uri; use pyrefly_build::handle::Handle; +use pyrefly_python::ast::Ast; use pyrefly_python::module::Module; use pyrefly_python::module_name::ModuleName; +use pyrefly_python::module_path::ModulePath; +use pyrefly_python::module_path::ModulePathDetails; +use pyrefly_python::symbol_kind::SymbolKind; +use pyrefly_python::sys_info::SysInfo; use pyrefly_util::absolutize::Absolutize; use pyrefly_util::fs_anyhow; use pyrefly_util::lined_buffer::DisplayPos; use pyrefly_util::lined_buffer::DisplayRange; use pyrefly_util::unix_path::path_to_unix_string; +use pyrefly_util::visit::Visit; +use ruff_python_ast::Expr; +use ruff_python_ast::ExprContext; +use ruff_python_ast::ExprName; +use ruff_python_ast::name::Name; use ruff_text_size::TextRange; use scip::symbol::format_symbol; use scip::types::Descriptor; @@ -29,11 +42,13 @@ use scip::types::SymbolRole; use scip::types::TextEncoding; use scip::types::ToolInfo; use scip::types::descriptor::Suffix; +use starlark_map::small_map::SmallMap; use crate::config::config::ConfigFile; use crate::report::scip::symbols::ScipSymbol; use crate::report::scip::symbols::definitions; use crate::report::scip::symbols::descriptor; +use crate::state::lsp::FindPreference; use crate::state::state::Transaction; #[cfg(test)] @@ -87,9 +102,15 @@ pub fn scip( version: &str, ) -> Index { let project_root = project_root.absolutize(); + let mut indexer = Indexer { + transaction, + project_root: &project_root, + project, + definitions: HashMap::new(), + }; let mut documents: Vec = handles .iter() - .filter_map(|handle| document(transaction, handle, &project_root, project)) + .filter_map(|handle| indexer.document(handle)) .collect(); documents.sort_by(|a, b| a.relative_path.cmp(&b.relative_path)); Index { @@ -112,38 +133,141 @@ pub fn scip( } } -fn document( - transaction: &Transaction, - handle: &Handle, - project_root: &Path, - project: &Package, -) -> Option { - let module = transaction.get_module_info(handle)?; - let relative_path = module.path().as_path().strip_prefix(project_root).ok()?; - let ast = transaction.get_ast(handle)?; - let mut occurrences = vec![Occurrence { - range: vec![0, 0, 0], - symbol: module_symbol(project, module.name()), - symbol_roles: SymbolRole::Definition as i32, - ..Default::default() - }]; - occurrences.extend( - definitions(module.name(), &ast) - .into_iter() - .map(|(range, symbol)| Occurrence { - range: scip_range(&module, range), - symbol: format_scip_symbol(project, &symbol), - symbol_roles: SymbolRole::Definition as i32, +struct Indexer<'a> { + transaction: &'a Transaction<'a>, + project_root: &'a Path, + project: &'a Package, + /// The definitions of every module seen so far, keyed by the range of the defining identifier. + definitions: HashMap>>, +} + +impl Indexer<'_> { + fn document(&mut self, handle: &Handle) -> Option { + let module = self.transaction.get_module_info(handle)?; + let relative_path = module + .path() + .as_path() + .strip_prefix(self.project_root) + .ok()?; + let ast = self.transaction.get_ast(handle)?; + let definitions = self.definitions(&module, handle.sys_info().dupe())?; + let mut occurrences = vec![Occurrence { + range: vec![0, 0, 0], + symbol: module_symbol(self.project, module.name()), + symbol_roles: SymbolRole::Definition as i32, + ..Default::default() + }]; + occurrences.extend(definitions.iter().map(|(range, symbol)| Occurrence { + range: scip_range(&module, *range), + symbol: format_scip_symbol(self.project, symbol), + symbol_roles: SymbolRole::Definition as i32, + ..Default::default() + })); + + // Names that resolve to no definition get one local per name, numbered after the + // module's own locals. + let first_unresolved = definitions + .values() + .filter_map(|symbol| match symbol { + ScipSymbol::Local(id) => Some(id + 1), + ScipSymbol::Global(_) => None, + }) + .max() + .unwrap_or(0); + let mut unresolved: SmallMap = SmallMap::new(); + let mut names = Vec::new(); + ast.visit(&mut |x| collect_name_uses(x, &mut names)); + for name in names { + let symbol = self.reference(handle, &module, name).unwrap_or_else(|| { + let id = first_unresolved + unresolved.len(); + format!("local {}", unresolved.entry(name.id.clone()).or_insert(id)) + }); + occurrences.push(Occurrence { + range: scip_range(&module, name.range), + symbol, ..Default::default() - }), - ); - Some(Document { - language: "python".to_owned(), - relative_path: path_to_unix_string(relative_path), - position_encoding: PositionEncoding::UTF32CodeUnitOffsetFromLineStart.into(), - occurrences, - ..Default::default() - }) + }); + } + Some(Document { + language: "python".to_owned(), + relative_path: path_to_unix_string(relative_path), + position_encoding: PositionEncoding::UTF32CodeUnitOffsetFromLineStart.into(), + occurrences, + ..Default::default() + }) + } + + /// The symbol a name use refers to, or `None` if it has no definition we can name. + fn reference(&mut self, handle: &Handle, module: &Module, name: &ExprName) -> Option { + let preference = FindPreference { + // A call like `C()` refers to the class `C`, not to `C.__init__`. + resolve_call_dunders: false, + ..Default::default() + }; + let definition = self + .transaction + .find_definition_for_name_use( + handle, + &Ast::expr_name_identifier(name.clone()), + preference, + ) + .ok()??; + let package = self.package(&definition.module, handle.sys_info().dupe()); + if definition.metadata.symbol_kind() == Some(SymbolKind::Module) { + return Some(module_symbol(&package, definition.module.name())); + } + let symbols = self.definitions(&definition.module, handle.sys_info().dupe())?; + match symbols.get(&definition.definition_range)? { + ScipSymbol::Global(descriptors) => Some(global_symbol(&package, descriptors.clone())), + ScipSymbol::Local(id) => { + (definition.module.path() == module.path()).then(|| format!("local {id}")) + } + } + } + + fn definitions( + &mut self, + module: &Module, + sys_info: SysInfo, + ) -> Option>> { + if let Some(definitions) = self.definitions.get(module.path()) { + return Some(definitions.dupe()); + } + let handle = Handle::new(module.name(), module.path().dupe(), sys_info); + let ast = self.transaction.get_ast(&handle)?; + let table: Arc> = + Arc::new(definitions(module.name(), &ast).into_iter().collect()); + self.definitions.insert(module.path().dupe(), table.dupe()); + Some(table) + } + + fn package(&self, module: &Module, sys_info: SysInfo) -> Package { + match module.path().details() { + ModulePathDetails::BundledTypeshed(_) => { + let version = sys_info.version(); + Package { + name: "python-stdlib".to_owned(), + version: format!("{}.{}", version.major, version.minor), + } + } + _ if module.path().as_path().starts_with(self.project_root) => self.project.clone(), + // Without package metadata, name the package after the top-level module. + _ => Package { + name: module.name().first_component().to_string(), + version: String::new(), + }, + } + } +} + +/// Collects every name that is read or deleted, in source order. +fn collect_name_uses<'a>(x: &'a Expr, names: &mut Vec<&'a ExprName>) { + if let Expr::Name(name) = x + && name.ctx != ExprContext::Store + { + names.push(name); + } + x.recurse(&mut |x| collect_name_uses(x, names)); } /// A SCIP range: `[line, start, end]` on one line, or `[start line, start, end line, end]`. @@ -201,8 +325,6 @@ fn module_symbol(package: &Package, module: ModuleName) -> String { mod tests { use std::path::PathBuf; - use pyrefly_python::module_path::ModulePath; - use super::*; use crate::report::scip::snapshot::format_snapshot; use crate::state::require::Require; diff --git a/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py index e4cb829184..0b183e1b3a 100644 --- a/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py +++ b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py @@ -4,4 +4,6 @@ import aliased as A print(A.SOME_CONSTANT) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^ reference snapshot-util 0.1 aliased/__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py index e17dd7242d..c986e52a8f 100644 --- a/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py +++ b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py @@ -6,4 +6,8 @@ from typing import Callable, Optional print(re, Callable, Optional) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^ reference python-stdlib 3.13 re/__init__: +# ^^^^^^^^ reference python-stdlib 3.13 typing/Callable. +# ^^^^^^^^ reference python-stdlib 3.13 typing/Optional. diff --git a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py index e2a576a334..49d0a6f315 100644 --- a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py +++ b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py @@ -7,19 +7,30 @@ class Example: # Note, only y has a type hint y: int # ^ definition snapshot-util 0.1 class_nohint/Example#y. +# ^^^ reference python-stdlib 3.13 builtins/int# def __init__(self, in_val): # ^^^^^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__(). # ^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__().(self) # ^^^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__().(in_val) self.x = in_val +# ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) # ^ definition snapshot-util 0.1 class_nohint/Example#x. +# ^^^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(in_val) self.x = self.x + 1 +# ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) +# ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) self.y = in_val +# ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) +# ^^^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(in_val) def something(self): # ^^^^^^^^^ definition snapshot-util 0.1 class_nohint/Example#something(). # ^^^^ definition snapshot-util 0.1 class_nohint/Example#something().(self) print(self.x) +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^ reference snapshot-util 0.1 class_nohint/Example#something().(self) print(self.y) +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^ reference snapshot-util 0.1 class_nohint/Example#something().(self) diff --git a/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py index c4d692baae..15b9f6db1b 100644 --- a/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py +++ b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py @@ -4,28 +4,44 @@ def something(x): # ^^^^^^^^^ definition snapshot-util 0.1 comp/something(). # ^ definition snapshot-util 0.1 comp/something().(x) return 2 * x +# ^ reference snapshot-util 0.1 comp/something().(x) _ = [x for x in "should be local 0"] # definition snapshot-util 0.1 comp/_. +# ^ reference local 0 # ^ definition local 0 _ = [something(x) for x in "should be local 1"] # definition snapshot-util 0.1 comp/_. +# ^^^^^^^^^ reference snapshot-util 0.1 comp/something(). +# ^ reference local 1 # ^ definition local 1 _ = [something(x) for x in "should be local 2" if x == "s"] # definition snapshot-util 0.1 comp/_. +# ^^^^^^^^^ reference snapshot-util 0.1 comp/something(). +# ^ reference local 2 # ^ definition local 2 +# ^ reference local 2 _ = {k: x for (k, x) in enumerate(range(10))} # definition snapshot-util 0.1 comp/_. +# ^ reference local 3 +# ^ reference local 4 # ^ definition local 3 # ^ definition local 4 +# ^^^^^^^^^ reference python-stdlib 3.13 builtins/enumerate# +# ^^^^^ reference python-stdlib 3.13 builtins/range# asdf = (var for var in [1, 2, 3] if var % 2 == 0) #^^^ definition snapshot-util 0.1 comp/asdf. +# ^^^ reference local 5 # ^^^ definition local 5 +# ^^^ reference local 5 for var in asdf: # ^^^ definition snapshot-util 0.1 comp/var. +# ^^^^ reference snapshot-util 0.1 comp/asdf. print(var) +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^ reference snapshot-util 0.1 comp/var. # TODO: ListComprehensions with if # TODO: Dictionary comprehensions diff --git a/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py b/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py index cb69e1c3cb..809609201e 100644 --- a/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py +++ b/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py @@ -2,6 +2,8 @@ if __name__ == '__main__': +# ^^^^^^^^ reference local 0 print("main") +# ^^^^^ reference python-stdlib 3.13 builtins/print(). diff --git a/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py b/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py index fd7495a896..a6e76749f0 100644 --- a/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/empty_project_name_version/main.py @@ -5,4 +5,5 @@ def main(): pass main() +#^^^ reference . . main/main(). diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py index 151160c7ad..4b1a61955c 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py @@ -9,20 +9,30 @@ def wrapper(*args, **kwargs): # ^^^^ definition snapshot-util 0.1 advanced/class_decorator().wrapper().(args) # ^^^^^^ definition snapshot-util 0.1 advanced/class_decorator().wrapper().(kwargs) return cls(*args, **kwargs) +# ^^^ reference snapshot-util 0.1 advanced/class_decorator().(cls) +# ^^^^ reference snapshot-util 0.1 advanced/class_decorator().wrapper().(args) +# ^^^^^^ reference snapshot-util 0.1 advanced/class_decorator().wrapper().(kwargs) return wrapper +# ^^^^^^^ reference snapshot-util 0.1 advanced/class_decorator().wrapper(). @class_decorator +#^^^^^^^^^^^^^^^ reference snapshot-util 0.1 advanced/class_decorator(). class Test: # ^^^^ definition snapshot-util 0.1 advanced/Test# def __init__(self, x: float): # ^^^^^^^^ definition snapshot-util 0.1 advanced/Test#__init__(). # ^^^^ definition snapshot-util 0.1 advanced/Test#__init__().(self) # ^ definition snapshot-util 0.1 advanced/Test#__init__().(x) +# ^^^^^ reference python-stdlib 3.13 builtins/float# self.x = x +# ^^^^ reference snapshot-util 0.1 advanced/Test#__init__().(self) # ^ definition snapshot-util 0.1 advanced/Test#x. +# ^ reference snapshot-util 0.1 advanced/Test#__init__().(x) def test(self) -> float: # ^^^^ definition snapshot-util 0.1 advanced/Test#test(). # ^^^^ definition snapshot-util 0.1 advanced/Test#test().(self) +# ^^^^^ reference python-stdlib 3.13 builtins/float# return self.x +# ^^^^ reference snapshot-util 0.1 advanced/Test#test().(self) diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py index 1c7632d25d..36d99e0482 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py @@ -9,14 +9,23 @@ def wrapper(*args, **kwargs): # ^^^^ definition snapshot-util 0.1 advanced/decorator().wrapper().(args) # ^^^^^^ definition snapshot-util 0.1 advanced/decorator().wrapper().(kwargs) return func(*args, **kwargs) +# ^^^^ reference snapshot-util 0.1 advanced/decorator().(func) +# ^^^^ reference snapshot-util 0.1 advanced/decorator().wrapper().(args) +# ^^^^^^ reference snapshot-util 0.1 advanced/decorator().wrapper().(kwargs) return wrapper +# ^^^^^^^ reference snapshot-util 0.1 advanced/decorator().wrapper(). @decorator +#^^^^^^^^^ reference snapshot-util 0.1 advanced/decorator(). def func(x: float) -> float: # ^^^^ definition snapshot-util 0.1 advanced/func(). # ^ definition snapshot-util 0.1 advanced/func().(x) +# ^^^^^ reference python-stdlib 3.13 builtins/float# +# ^^^^^ reference python-stdlib 3.13 builtins/float# test = x # ^^^^ definition local 0 +# ^ reference snapshot-util 0.1 advanced/func().(x) return test +# ^^^^ reference local 0 diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py index 37e9f8e6b0..f845d5e004 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py @@ -5,4 +5,5 @@ def simple(x): # ^^^^^^ definition snapshot-util 0.1 simple/simple(). # ^ definition snapshot-util 0.1 simple/simple().(x) return x +# ^ reference snapshot-util 0.1 simple/simple().(x) diff --git a/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py b/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py index 88ec7923b3..def92e6b15 100644 --- a/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/f_string/fstring.py @@ -4,4 +4,6 @@ #^^ definition snapshot-util 0.1 fstring/var. print(f"var: hello {var}") +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^ reference snapshot-util 0.1 fstring/var. diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py index c05cd172e1..d166944044 100644 --- a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py @@ -3,4 +3,6 @@ from xyz import nested_file print(nested_file.X) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py index 223f9c47cb..83c67081b1 100644 --- a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py @@ -3,4 +3,6 @@ from xyz import nested_file print(nested_file.X) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py index a4b697558f..e03b250d0d 100644 --- a/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py +++ b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py @@ -4,32 +4,49 @@ _T_co = TypeVar("_T_co") #^^^^ definition snapshot-util 0.1 mwe/_T_co. +# ^^^^^^^ reference python-stdlib 3.13 typing/TypeVar# _P = ParamSpec("_P") #^ definition snapshot-util 0.1 mwe/_P. +# ^^^^^^^^^ reference python-stdlib 3.13 typing/ParamSpec# class X(Generic[_T_co]): # ^ definition snapshot-util 0.1 mwe/X# +# ^^^^^^^ reference python-stdlib 3.13 typing/Generic. +# ^^^^^ reference snapshot-util 0.1 mwe/_T_co. pass def decorate(func: Callable[_P, Iterator[_T_co]]) -> Callable[_P, X[_T_co]]: ... # ^^^^^^^^ definition snapshot-util 0.1 mwe/decorate(). # ^^^^ definition snapshot-util 0.1 mwe/decorate().(func) +# ^^^^^^^^ reference python-stdlib 3.13 typing/Callable. +# ^^ reference snapshot-util 0.1 mwe/_P. +# ^^^^^^^^ reference python-stdlib 3.13 typing/Iterator# +# ^^^^^ reference snapshot-util 0.1 mwe/_T_co. +# ^^^^^^^^ reference python-stdlib 3.13 typing/Callable. +# ^^ reference snapshot-util 0.1 mwe/_P. +# ^ reference snapshot-util 0.1 mwe/X# +# ^^^^^ reference snapshot-util 0.1 mwe/_T_co. class Foo: # ^^^ definition snapshot-util 0.1 mwe/Foo# @decorate +# ^^^^^^^^ reference snapshot-util 0.1 mwe/decorate(). def foo(self) -> Iterator[None]: ... # ^^^ definition snapshot-util 0.1 mwe/Foo#foo(). # ^^^^ definition snapshot-util 0.1 mwe/Foo#foo().(self) +# ^^^^^^^^ reference python-stdlib 3.13 typing/Iterator# @decorate +#^^^^^^^^ reference snapshot-util 0.1 mwe/decorate(). def noop(): # ^^^^ definition snapshot-util 0.1 mwe/noop(). yield class FooImpl(Foo): # ^^^^^^^ definition snapshot-util 0.1 mwe/FooImpl# +# ^^^ reference snapshot-util 0.1 mwe/Foo# def foo(self): # ^^^ definition snapshot-util 0.1 mwe/FooImpl#foo(). # ^^^^ definition snapshot-util 0.1 mwe/FooImpl#foo().(self) return noop() +# ^^^^ reference snapshot-util 0.1 mwe/noop(). diff --git a/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py index ea00e7f72f..5b9501ee04 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py +++ b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py @@ -3,5 +3,7 @@ import importlib.resources importlib.resources.read_text('pre_commit.resources', 'filename') +#^^^^^^^^ reference python-stdlib 3.13 importlib/__init__: importlib.resources.read_text('pre_commit.resources', 'filename') +#^^^^^^^^ reference python-stdlib 3.13 importlib/__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py index 460d346aa6..1baa1c8466 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py @@ -4,9 +4,11 @@ class SuchNestedMuchWow: # ^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow# class_item: int = 42 # ^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow#class_item. +# ^^^ reference python-stdlib 3.13 builtins/int# class AnotherNestedMuchWow: # ^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow# other_item: int = 42 # ^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow#other_item. +# ^^^ reference python-stdlib 3.13 builtins/int# diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py index 871db5e815..11312462fb 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py @@ -4,6 +4,12 @@ from foo.bar.baz.mod import SuchNestedMuchWow, AnotherNestedMuchWow print(SuchNestedMuchWow().class_item) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^^^^^^^^^^ reference local 0 print(AnotherNestedMuchWow().other_item) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^^^^^^^^^^^^^ reference local 1 print(InitClass().init_item) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^^ reference local 2 diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py index 8c157c0059..5e3100fd56 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py @@ -3,4 +3,6 @@ import foo.bar.baz.mod print(foo.bar.baz.mod.SuchNestedMuchWow) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^ reference local 0 diff --git a/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py b/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py index 32d5f03bf3..e732449a4e 100644 --- a/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/odd_pkg_name_1/main.py @@ -5,4 +5,5 @@ def main(): pass main() +#^^^ reference package with space 0.1 main/main(). diff --git a/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py index f404c12282..eb2e64294d 100644 --- a/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/pyproject_1/main.py @@ -5,4 +5,5 @@ def main(): pass main() +#^^^ reference abc 16.05 main/main(). diff --git a/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py index 8811594b77..7c044c5993 100644 --- a/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py +++ b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py @@ -5,6 +5,8 @@ if True: exported_function() +# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 foo/exported_function(). this_class.exported_function() +# ^^^^^^^^^^ reference snapshot-util 0.1 foo/this_class. diff --git a/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py b/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py index e2a2bd044a..3bbb6c7331 100644 --- a/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py +++ b/pyrefly/lib/report/scip/snapshots/output/relative_import/foo.py @@ -20,4 +20,5 @@ def exported_function(self): this_class = MyClass() #^^^^^^^^^ definition snapshot-util 0.1 foo/this_class. +# ^^^^^^^ reference snapshot-util 0.1 foo/MyClass# diff --git a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py index 0a475374ba..17c00f32f9 100644 --- a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py @@ -4,10 +4,13 @@ class ExampleClass: # ^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass# a: int # ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#a. +# ^^^ reference python-stdlib 3.13 builtins/int# b: int # ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#b. +# ^^^ reference python-stdlib 3.13 builtins/int# c: str # ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#c. +# ^^^ reference python-stdlib 3.13 builtins/str# static_var = "Hello World" # ^^^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#static_var. @@ -16,11 +19,19 @@ def __init__(self, a: int, b: int): # ^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__(). # ^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) # ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(a) +# ^^^ reference python-stdlib 3.13 builtins/int# # ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(b) +# ^^^ reference python-stdlib 3.13 builtins/int# local_c = ", world!" # ^^^^^^^ definition local 0 self.a = a +# ^^^^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) +# ^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(a) self.b = b +# ^^^^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) +# ^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(b) self.c = "hello" + local_c +# ^^^^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) +# ^^^^^^^ reference local 0 diff --git a/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py index d6b65e1f55..87eb165a77 100644 --- a/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py +++ b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py @@ -3,22 +3,33 @@ def my_cool_function(a: str) -> str: # ^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/my_cool_function(). # ^ definition snapshot-util 0.1 `src.single_function`/my_cool_function().(a) +# ^^^ reference python-stdlib 3.13 builtins/str# +# ^^^ reference python-stdlib 3.13 builtins/str# x = ", world" # ^ definition local 0 return a + x +# ^ reference snapshot-util 0.1 `src.single_function`/my_cool_function().(a) +# ^ reference local 0 def my_cool_function_2(a: str): # ^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/my_cool_function_2(). # ^ definition snapshot-util 0.1 `src.single_function`/my_cool_function_2().(a) +# ^^^ reference python-stdlib 3.13 builtins/str# x = ", world" # ^ definition local 1 return (lambda y: a + x + y)("oh no") # ^ definition local 2 +# ^ reference snapshot-util 0.1 `src.single_function`/my_cool_function_2().(a) +# ^ reference local 1 +# ^ reference local 3 def next_level(): # ^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/next_level(). return my_cool_function +# ^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `src.single_function`/my_cool_function(). my_cool_function("hello") +#^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `src.single_function`/my_cool_function(). next_level()(a = "wow") +#^^^^^^^^^ reference snapshot-util 0.1 `src.single_function`/next_level(). diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py index 10fb85dbf3..583bfdaa0d 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py @@ -3,4 +3,6 @@ from typing import Any print(Any) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^ reference python-stdlib 3.13 typing/Any# diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py index 0db4f3d978..66fdcc5e18 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py @@ -7,5 +7,6 @@ class ClassWithField: a: int # ^ definition snapshot-util 0.1 field_docstring/ClassWithField#a. +# ^^^ reference python-stdlib 3.13 builtins/int# """Hello world, this is a""" diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py index d1a56b2712..7c8fd05793 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py @@ -5,16 +5,20 @@ class A: def grandparent(self) -> bool: # ^^^^^^^^^^^ definition snapshot-util 0.1 grandparent_impl/A#grandparent(). # ^^^^ definition snapshot-util 0.1 grandparent_impl/A#grandparent().(self) +# ^^^^ reference python-stdlib 3.13 builtins/bool# return True class B(A): # ^ definition snapshot-util 0.1 grandparent_impl/B# +# ^ reference snapshot-util 0.1 grandparent_impl/A# ... class C(B): # ^ definition snapshot-util 0.1 grandparent_impl/C# +# ^ reference snapshot-util 0.1 grandparent_impl/B# def grandparent(self) -> bool: # ^^^^^^^^^^^ definition snapshot-util 0.1 grandparent_impl/C#grandparent(). # ^^^^ definition snapshot-util 0.1 grandparent_impl/C#grandparent().(self) +# ^^^^ reference python-stdlib 3.13 builtins/bool# return False diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py index c11f2db9db..af26724502 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py @@ -8,6 +8,9 @@ def __init__(self, b: int): # ^^^^^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__(). # ^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(self) # ^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(b) +# ^^^ reference python-stdlib 3.13 builtins/int# self.b = b +# ^^^^ reference snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(self) # ^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#b. +# ^ reference snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(b) diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py index d3f6ba18db..f6394ab6db 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py @@ -5,26 +5,33 @@ class A: def x(self) -> int: # ^ definition snapshot-util 0.1 inherits_class/A#x(). # ^^^^ definition snapshot-util 0.1 inherits_class/A#x().(self) +# ^^^ reference python-stdlib 3.13 builtins/int# raise NotImplemented +# ^^^^^^^^^^^^^^ reference python-stdlib 3.13 builtins/NotImplemented. def matched_despite_different_type(self, x: int): # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inherits_class/A#matched_despite_different_type(). # ^^^^ definition snapshot-util 0.1 inherits_class/A#matched_despite_different_type().(self) # ^ definition snapshot-util 0.1 inherits_class/A#matched_despite_different_type().(x) +# ^^^ reference python-stdlib 3.13 builtins/int# pass class B(A): # ^ definition snapshot-util 0.1 inherits_class/B# +# ^ reference snapshot-util 0.1 inherits_class/A# def x(self) -> int: # ^ definition snapshot-util 0.1 inherits_class/B#x(). # ^^^^ definition snapshot-util 0.1 inherits_class/B#x().(self) +# ^^^ reference python-stdlib 3.13 builtins/int# return 5 def matched_despite_different_type(self, x: int, y: int): # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type(). # ^^^^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(self) # ^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(x) +# ^^^ reference python-stdlib 3.13 builtins/int# # ^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(y) +# ^^^ reference python-stdlib 3.13 builtins/int# pass def unrelated(self): diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py index a31e819920..fd38f83aa7 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py @@ -5,11 +5,13 @@ class Left: def one(self) -> int: # ^^^ definition snapshot-util 0.1 multiinherits_test/Left#one(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Left#one().(self) +# ^^^ reference python-stdlib 3.13 builtins/int# return 1 def shared(self) -> bool: # ^^^^^^ definition snapshot-util 0.1 multiinherits_test/Left#shared(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Left#shared().(self) +# ^^^^ reference python-stdlib 3.13 builtins/bool# return False class Right: @@ -22,13 +24,17 @@ def two(self): def shared(self) -> bool: # ^^^^^^ definition snapshot-util 0.1 multiinherits_test/Right#shared(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Right#shared().(self) +# ^^^^ reference python-stdlib 3.13 builtins/bool# return False class Multi(Left, Right): # ^^^^^ definition snapshot-util 0.1 multiinherits_test/Multi# +# ^^^^ reference snapshot-util 0.1 multiinherits_test/Left# +# ^^^^^ reference snapshot-util 0.1 multiinherits_test/Right# def one(self) -> int: # ^^^ definition snapshot-util 0.1 multiinherits_test/Multi#one(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#one().(self) +# ^^^ reference python-stdlib 3.13 builtins/int# return 1 def two(self): @@ -44,5 +50,6 @@ def three(self): def shared(self) -> bool: # ^^^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#shared(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#shared().(self) +# ^^^^ reference python-stdlib 3.13 builtins/bool# return True diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py index 60b24e7200..ec49e3c6a6 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py @@ -10,6 +10,7 @@ def __init__(self): pass @property +# ^^^^^^^^ reference python-stdlib 3.13 builtins/property# def prop_ref(self): # ^^^^^^^^ definition snapshot-util 0.1 property_access/PropertyClass#prop_ref(). # ^^^^ definition snapshot-util 0.1 property_access/PropertyClass#prop_ref().(self) @@ -18,13 +19,19 @@ def prop_ref(self): xs = [PropertyClass()] #^ definition snapshot-util 0.1 property_access/xs. +# ^^^^^^^^^^^^^ reference snapshot-util 0.1 property_access/PropertyClass# def usage(xs: Sequence[PropertyClass]): # ^^^^^ definition snapshot-util 0.1 property_access/usage(). # ^^ definition snapshot-util 0.1 property_access/usage().(xs) +# ^^^^^^^^ reference python-stdlib 3.13 typing/Sequence# +# ^^^^^^^^^^^^^ reference snapshot-util 0.1 property_access/PropertyClass# def nested(): # ^^^^^^ definition snapshot-util 0.1 property_access/usage().nested(). for x in xs: # ^ definition local 0 +# ^^ reference snapshot-util 0.1 property_access/usage().(xs) print(x.prop_ref) +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^ reference local 0 diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py index 029b95563c..cead9649f4 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py @@ -6,16 +6,21 @@ class X: # ^ definition snapshot-util 0.1 vars_inside_scopes/X# items: List[int] # ^^^^^ definition snapshot-util 0.1 vars_inside_scopes/X#items. +# ^^^^ reference python-stdlib 3.13 typing/List. +# ^^^ reference python-stdlib 3.13 builtins/int# def my_func(self): # ^^^^^^^ definition snapshot-util 0.1 vars_inside_scopes/X#my_func(). # ^^^^ definition snapshot-util 0.1 vars_inside_scopes/X#my_func().(self) for x in self.items: # ^ definition local 0 +# ^^^^ reference snapshot-util 0.1 vars_inside_scopes/X#my_func().(self) y = x + 1 # ^ definition local 1 +# ^ reference local 0 if 5 in self.items: +# ^^^^ reference snapshot-util 0.1 vars_inside_scopes/X#my_func().(self) z = "oh ya" # ^ definition local 2 diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py index 5654899dbf..02f69f4896 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py @@ -12,30 +12,53 @@ # if statement if (n := len(some_list)) > 10: # ^ definition snapshot-util 0.1 walrus/n. +# ^^^ reference python-stdlib 3.13 builtins/len(). +# ^^^^^^^^^ reference snapshot-util 0.1 walrus/some_list. print(f"List is too long with {n} elements.") +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^ reference snapshot-util 0.1 walrus/n. if (match := re.search(pattern, text)) is not None: # ^^^^^ definition snapshot-util 0.1 walrus/match. +# ^^ reference python-stdlib 3.13 re/__init__: +# ^^^^^^^ reference snapshot-util 0.1 walrus/pattern. +# ^^^^ reference snapshot-util 0.1 walrus/text. print("Found match:", match.group(0)) +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^ reference snapshot-util 0.1 walrus/match. # comprehensions def show_some_comprehension(): # ^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 walrus/show_some_comprehension(). [print(x, root) for x in some_list if (x % 2 == 0) and (root := x**0.5) > 5] +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^ reference local 0 +# ^^^^ reference local 1 # ^ definition local 0 +# ^^^^^^^^^ reference snapshot-util 0.1 walrus/some_list. +# ^ reference local 0 # ^^^^ definition local 1 +# ^ reference local 0 # while loop while (line := input("Enter text: ")) != "quit": # ^^^^ definition snapshot-util 0.1 walrus/line. +# ^^^^^ reference python-stdlib 3.13 builtins/input(). print(f"You entered: {line}") +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^ reference snapshot-util 0.1 walrus/line. # if + comprehension if any((any_n := num) < 0 for num in some_list): +# ^^^ reference python-stdlib 3.13 builtins/any(). # ^^^^^ definition snapshot-util 0.1 walrus/any_n. +# ^^^ reference local 2 # ^^^ definition local 2 +# ^^^^^^^^^ reference snapshot-util 0.1 walrus/some_list. print(f"Negative number found: {any_n}") +# ^^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^ reference snapshot-util 0.1 walrus/any_n. diff --git a/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py b/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py index 3d890deb1d..95d33f3426 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py +++ b/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py @@ -3,6 +3,10 @@ import this_is_not_real print(this_is_not_real.x) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^^^^^^^^^ reference local 0 print(this_is_not_real.x) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^^^^^^^^^ reference local 0 From 53c45e646e6e9a0ca77236078ece9de6fea53f5a Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sun, 18 Oct 2026 21:24:19 -0400 Subject: [PATCH 06/21] handle imports --- pyrefly/lib/report/scip.rs | 195 +++++++++++++++--- .../input/request_goofiness/goofy.py | 3 + .../snapshots/output/aliased_import/actual.py | 3 + .../output/builtin_imports/builtin_imports.py | 4 + .../file_from_module_import/abc/file.py | 2 + .../file_from_module.py | 2 + .../scip/snapshots/output/graph_1278/mwe.py | 6 + .../output/nameparts_import/nameparts.py | 1 + .../output/nested_items/src/__init__.py | 2 +- .../output/nested_items/src/foo/__init__.py | 2 +- .../nested_items/src/foo/bar/__init__.py | 6 +- .../nested_items/src/foo/bar/baz/mod.py | 10 +- .../output/nested_items/src/importer.py | 13 +- .../output/nested_items/src/long_importer.py | 5 +- .../snapshots/output/relative_import/bar.py | 4 + .../output/request_goofiness/goofy.py | 9 + .../output/single_class/src/single_class.py | 30 +-- .../single_function/src/single_function.py | 22 +- .../output/unique/builtin_import_refs.py | 2 + .../output/unique/property_access.py | 2 + .../output/unique/vars_inside_scopes.py | 2 + .../scip/snapshots/output/unique/walrus.py | 1 + .../output/unresolved_import/unresolved.py | 1 + .../requests-2.0.0.dist-info/METADATA | 3 + .../requests-2.0.0.dist-info/RECORD | 3 + .../site-packages/requests/__init__.py | 0 26 files changed, 257 insertions(+), 76 deletions(-) create mode 100644 pyrefly/lib/report/scip/snapshots/input/request_goofiness/goofy.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py create mode 100644 pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/METADATA create mode 100644 pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/RECORD create mode 100644 pyrefly/lib/report/scip/snapshots/site-packages/requests/__init__.py diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index f8768950f6..62a5df8b5a 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -6,7 +6,9 @@ */ use std::collections::HashMap; +use std::iter; use std::path::Path; +use std::path::PathBuf; use std::sync::Arc; use dupe::Dupe; @@ -27,9 +29,12 @@ use pyrefly_util::unix_path::path_to_unix_string; use pyrefly_util::visit::Visit; use ruff_python_ast::Expr; use ruff_python_ast::ExprContext; -use ruff_python_ast::ExprName; +use ruff_python_ast::Identifier; +use ruff_python_ast::Stmt; use ruff_python_ast::name::Name; +use ruff_text_size::Ranged; use ruff_text_size::TextRange; +use ruff_text_size::TextSize; use scip::symbol::format_symbol; use scip::types::Descriptor; use scip::types::Document; @@ -107,6 +112,7 @@ pub fn scip( project_root: &project_root, project, definitions: HashMap::new(), + distributions: HashMap::new(), }; let mut documents: Vec = handles .iter() @@ -138,7 +144,9 @@ struct Indexer<'a> { project_root: &'a Path, project: &'a Package, /// The definitions of every module seen so far, keyed by the range of the defining identifier. - definitions: HashMap>>, + definitions: HashMap<(ModuleName, ModulePath), Arc>>, + /// The installed distributions of every site-packages directory seen so far. + distributions: HashMap>, } impl Indexer<'_> { @@ -175,15 +183,17 @@ impl Indexer<'_> { .max() .unwrap_or(0); let mut unresolved: SmallMap = SmallMap::new(); - let mut names = Vec::new(); - ast.visit(&mut |x| collect_name_uses(x, &mut names)); - for name in names { - let symbol = self.reference(handle, &module, name).unwrap_or_else(|| { + let mut uses = Vec::new(); + ast.visit(&mut |x| collect_name_uses(x, &mut uses)); + ast.body.iter().for_each(|x| collect_imports(x, &mut uses)); + uses.sort_by_key(|x| x.range.start()); + for x in uses { + let symbol = self.reference(handle, &module, &x).unwrap_or_else(|| { let id = first_unresolved + unresolved.len(); - format!("local {}", unresolved.entry(name.id.clone()).or_insert(id)) + format!("local {}", unresolved.entry(x.identifier.id).or_insert(id)) }); occurrences.push(Occurrence { - range: scip_range(&module, name.range), + range: scip_range(&module, x.range), symbol, ..Default::default() }); @@ -197,21 +207,27 @@ impl Indexer<'_> { }) } - /// The symbol a name use refers to, or `None` if it has no definition we can name. - fn reference(&mut self, handle: &Handle, module: &Module, name: &ExprName) -> Option { + /// The symbol a use refers to, or `None` if it has no definition we can name. + fn reference(&mut self, handle: &Handle, module: &Module, x: &Use) -> Option { let preference = FindPreference { // A call like `C()` refers to the class `C`, not to `C.__init__`. resolve_call_dunders: false, ..Default::default() }; - let definition = self - .transaction - .find_definition_for_name_use( - handle, - &Ast::expr_name_identifier(name.clone()), - preference, - ) - .ok()??; + let definition = if x.import { + // At the identifier's last character, a dotted module name resolves to the whole + // module rather than to its first component. + let position = x.identifier.range.end() - TextSize::from(1); + let definitions = self + .transaction + .find_definition(handle, position, preference) + .ok()?; + definitions.into_iter().next() + } else { + self.transaction + .find_definition_for_name_use(handle, &x.identifier, preference) + .ok()? + }?; let package = self.package(&definition.module, handle.sys_info().dupe()); if definition.metadata.symbol_kind() == Some(SymbolKind::Module) { return Some(module_symbol(&package, definition.module.name())); @@ -230,18 +246,19 @@ impl Indexer<'_> { module: &Module, sys_info: SysInfo, ) -> Option>> { - if let Some(definitions) = self.definitions.get(module.path()) { + let key = (module.name(), module.path().dupe()); + if let Some(definitions) = self.definitions.get(&key) { return Some(definitions.dupe()); } let handle = Handle::new(module.name(), module.path().dupe(), sys_info); let ast = self.transaction.get_ast(&handle)?; let table: Arc> = Arc::new(definitions(module.name(), &ast).into_iter().collect()); - self.definitions.insert(module.path().dupe(), table.dupe()); + self.definitions.insert(key, table.dupe()); Some(table) } - fn package(&self, module: &Module, sys_info: SysInfo) -> Package { + fn package(&mut self, module: &Module, sys_info: SysInfo) -> Package { match module.path().details() { ModulePathDetails::BundledTypeshed(_) => { let version = sys_info.version(); @@ -251,23 +268,129 @@ impl Indexer<'_> { } } _ if module.path().as_path().starts_with(self.project_root) => self.project.clone(), - // Without package metadata, name the package after the top-level module. - _ => Package { - name: module.name().first_component().to_string(), - version: String::new(), - }, + _ => { + // Match by top-level module, so that bundled stubs get the installed package too. + let top_level = module.name().first_component(); + let handle = Handle::new(module.name(), module.path().dupe(), sys_info); + let config = self.transaction.get_config(&handle); + config + .iter() + .flat_map(|config| config.site_package_path()) + .find_map(|root| { + self.distributions + .entry(root.clone()) + .or_insert_with(|| distributions(root)) + .get(&top_level) + .cloned() + }) + // Without package metadata, name the package after the top-level module. + .unwrap_or_else(|| Package { + name: top_level.to_string(), + version: String::new(), + }) + } + } + } +} + +/// The distributions installed in a site-packages directory, keyed by each top-level module they +/// installed. A distribution whose metadata can't be read is left out. +fn distributions(site_packages: &Path) -> HashMap { + let mut modules = HashMap::new(); + let pattern = site_packages.join("*.dist-info"); + for dist_info in glob::glob(&pattern.to_string_lossy()) + .into_iter() + .flatten() + .flatten() + { + let (Ok(metadata), Ok(record)) = ( + fs_anyhow::read_to_string(&dist_info.join("METADATA")), + fs_anyhow::read_to_string(&dist_info.join("RECORD")), + ) else { + continue; + }; + // The header fields end at the first blank line. + let field = |key: &str| { + metadata + .lines() + .take_while(|line| !line.is_empty()) + .find_map(|line| Some(line.strip_prefix(key)?.strip_prefix(':')?.trim().to_owned())) + }; + let (Some(name), Some(version)) = (field("Name"), field("Version")) else { + continue; + }; + let package = Package { name, version }; + // Each RECORD line is a CSV row whose first column is the installed file. Its first path + // component, up to the first `.`, names a top-level module (or a `-stubs` package), unless + // it isn't an identifier, as for the `.dist-info` directory itself. + for line in record.lines() { + let component = line.split(['/', ',']).next().unwrap_or_default(); + let module = component + .split('.') + .next() + .unwrap_or_default() + .trim_end_matches("-stubs"); + if !module.is_empty() && module.chars().all(|c| c.is_alphanumeric() || c == '_') { + modules + .entry(Name::new(module)) + .or_insert_with(|| package.clone()); + } } } + modules +} + +/// A name that refers to a definition elsewhere. +struct Use { + /// The range of the occurrence. For a relative module, it also covers the leading dots. + range: TextRange, + identifier: Identifier, + /// Whether the name is part of an import statement, rather than an expression. + import: bool, } -/// Collects every name that is read or deleted, in source order. -fn collect_name_uses<'a>(x: &'a Expr, names: &mut Vec<&'a ExprName>) { +/// Collects every name that is read or deleted. +fn collect_name_uses(x: &Expr, uses: &mut Vec) { if let Expr::Name(name) = x && name.ctx != ExprContext::Store { - names.push(name); + uses.push(Use { + range: name.range, + identifier: Ast::expr_name_identifier(name.clone()), + import: false, + }); + } + x.recurse(&mut |x| collect_name_uses(x, uses)); +} + +/// Collects the module and name identifiers of every import statement. +fn collect_imports(x: &Stmt, uses: &mut Vec) { + let mut import = |range, identifier: &Identifier| { + uses.push(Use { + range, + identifier: identifier.clone(), + import: true, + }) + }; + let aliases = match x { + Stmt::Import(x) => &x.names, + Stmt::ImportFrom(x) => { + if let Some(module) = &x.module { + let dots = TextSize::from(x.level); + import(TextRange::new(module.start() - dots, module.end()), module); + } + &x.names + } + _ => { + x.recurse(&mut |x| collect_imports(x, uses)); + return; + } + }; + for alias in aliases.iter().filter(|alias| alias.name.as_str() != "*") { + for identifier in iter::once(&alias.name).chain(&alias.asname) { + import(identifier.range, identifier); + } } - x.recurse(&mut |x| collect_name_uses(x, names)); } /// A SCIP range: `[line, start, end]` on one line, or `[start line, start, end line, end]`. @@ -323,8 +446,6 @@ fn module_symbol(package: &Package, module: ModuleName) -> String { #[cfg(test)] mod tests { - use std::path::PathBuf; - use super::*; use crate::report::scip::snapshot::format_snapshot; use crate::state::require::Require; @@ -361,8 +482,12 @@ mod tests { let mut handles = Vec::new(); for path in glob::glob(&format!("{}/**/*.py", input_dir.display())).unwrap() { let path = path.unwrap(); - let name = - ModuleName::from_relative_path(path.strip_prefix(&input_dir).unwrap()).unwrap(); + // As with an import root, modules under `src/` are named without the prefix. + let relative = path + .strip_prefix(input_dir.join("src")) + .or_else(|_| path.strip_prefix(&input_dir)) + .unwrap(); + let name = ModuleName::from_relative_path(relative).unwrap(); env.add_real_path(name.as_str(), path.clone()); handles.push(Handle::new( name, @@ -371,6 +496,7 @@ mod tests { )); } let (state, _) = env + .with_site_package_paths(vec![snapshots.join("site-packages")]) .with_default_require_level(Require::Everything) .to_state(); let index = scip( @@ -434,6 +560,7 @@ mod tests { odd_pkg_name_1, pyproject_1, relative_import, + request_goofiness, single_class, single_function, unique, diff --git a/pyrefly/lib/report/scip/snapshots/input/request_goofiness/goofy.py b/pyrefly/lib/report/scip/snapshots/input/request_goofiness/goofy.py new file mode 100644 index 0000000000..4862e66b1c --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/request_goofiness/goofy.py @@ -0,0 +1,3 @@ +import requests + +print(requests.get("https://sourcegraph.com")) diff --git a/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py index 0b183e1b3a..ed85a3d7d7 100644 --- a/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py +++ b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py @@ -1,7 +1,10 @@ # < definition scip-python python snapshot-util 0.1 actual/__init__: import aliased +# ^^^^^^^ reference snapshot-util 0.1 aliased/__init__: import aliased as A +# ^^^^^^^ reference snapshot-util 0.1 aliased/__init__: +# ^ reference snapshot-util 0.1 aliased/__init__: print(A.SOME_CONSTANT) #^^^^ reference python-stdlib 3.13 builtins/print(). diff --git a/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py index c986e52a8f..095c0589aa 100644 --- a/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py +++ b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py @@ -3,7 +3,11 @@ # format-options: showDocs import re +# ^^ reference python-stdlib 3.13 re/__init__: from typing import Callable, Optional +# ^^^^^^ reference python-stdlib 3.13 typing/__init__: +# ^^^^^^^^ reference python-stdlib 3.13 typing/Callable. +# ^^^^^^^^ reference python-stdlib 3.13 typing/Optional. print(re, Callable, Optional) #^^^^ reference python-stdlib 3.13 builtins/print(). diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py index d166944044..b9beb098c0 100644 --- a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py @@ -1,6 +1,8 @@ # < definition scip-python python snapshot-util 0.1 `abc.file`/__init__: from xyz import nested_file +# ^^^ reference snapshot-util 0.1 xyz/__init__: +# ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: print(nested_file.X) #^^^^ reference python-stdlib 3.13 builtins/print(). diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py index 83c67081b1..50feb26819 100644 --- a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py @@ -1,6 +1,8 @@ # < definition scip-python python snapshot-util 0.1 file_from_module/__init__: from xyz import nested_file +# ^^^ reference snapshot-util 0.1 xyz/__init__: +# ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: print(nested_file.X) #^^^^ reference python-stdlib 3.13 builtins/print(). diff --git a/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py index e03b250d0d..0e4078b562 100644 --- a/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py +++ b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py @@ -1,6 +1,12 @@ # < definition scip-python python snapshot-util 0.1 mwe/__init__: from typing import TypeVar, Generic, Callable, Iterator, ParamSpec +# ^^^^^^ reference python-stdlib 3.13 typing/__init__: +# ^^^^^^^ reference python-stdlib 3.13 typing/TypeVar# +# ^^^^^^^ reference python-stdlib 3.13 typing/Generic. +# ^^^^^^^^ reference python-stdlib 3.13 typing/Callable. +# ^^^^^^^^ reference python-stdlib 3.13 typing/Iterator# +# ^^^^^^^^^ reference python-stdlib 3.13 typing/ParamSpec# _T_co = TypeVar("_T_co") #^^^^ definition snapshot-util 0.1 mwe/_T_co. diff --git a/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py index 5b9501ee04..95a9753d3a 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py +++ b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py @@ -1,6 +1,7 @@ # < definition scip-python python snapshot-util 0.1 nameparts/__init__: import importlib.resources +# ^^^^^^^^^^^^^^^^^^^ reference python-stdlib 3.13 `importlib.resources`/__init__: importlib.resources.read_text('pre_commit.resources', 'filename') #^^^^^^^^ reference python-stdlib 3.13 importlib/__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py index 00291ef7ae..5da15c4a1e 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/__init__.py @@ -1,3 +1,3 @@ -# < definition scip-python python snapshot-util 0.1 src/__init__: +# < definition scip-python python snapshot-util 0.1 /__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py index 39b336df65..1bbbbacc1a 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/__init__.py @@ -1,3 +1,3 @@ -# < definition scip-python python snapshot-util 0.1 `src.foo`/__init__: +# < definition scip-python python snapshot-util 0.1 foo/__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py index 29a739c24e..7bac12003b 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/__init__.py @@ -1,7 +1,7 @@ -# < definition scip-python python snapshot-util 0.1 `src.foo.bar`/__init__: +# < definition scip-python python snapshot-util 0.1 `foo.bar`/__init__: class InitClass: -# ^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar`/InitClass# +# ^^^^^^^^^ definition snapshot-util 0.1 `foo.bar`/InitClass# init_item = 10 -# ^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar`/InitClass#init_item. +# ^^^^^^^^^ definition snapshot-util 0.1 `foo.bar`/InitClass#init_item. diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py index 1baa1c8466..713670fbeb 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/foo/bar/baz/mod.py @@ -1,14 +1,14 @@ -# < definition scip-python python snapshot-util 0.1 `src.foo.bar.baz.mod`/__init__: +# < definition scip-python python snapshot-util 0.1 `foo.bar.baz.mod`/__init__: class SuchNestedMuchWow: -# ^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow# +# ^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `foo.bar.baz.mod`/SuchNestedMuchWow# class_item: int = 42 -# ^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/SuchNestedMuchWow#class_item. +# ^^^^^^^^^^ definition snapshot-util 0.1 `foo.bar.baz.mod`/SuchNestedMuchWow#class_item. # ^^^ reference python-stdlib 3.13 builtins/int# class AnotherNestedMuchWow: -# ^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow# +# ^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `foo.bar.baz.mod`/AnotherNestedMuchWow# other_item: int = 42 -# ^^^^^^^^^^ definition snapshot-util 0.1 `src.foo.bar.baz.mod`/AnotherNestedMuchWow#other_item. +# ^^^^^^^^^^ definition snapshot-util 0.1 `foo.bar.baz.mod`/AnotherNestedMuchWow#other_item. # ^^^ reference python-stdlib 3.13 builtins/int# diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py index 11312462fb..dd48d8967b 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py @@ -1,15 +1,20 @@ -# < definition scip-python python snapshot-util 0.1 `src.importer`/__init__: +# < definition scip-python python snapshot-util 0.1 importer/__init__: from foo.bar import InitClass +# ^^^^^^^ reference snapshot-util 0.1 `foo.bar`/__init__: +# ^^^^^^^^^ reference snapshot-util 0.1 `foo.bar`/InitClass# from foo.bar.baz.mod import SuchNestedMuchWow, AnotherNestedMuchWow +# ^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/__init__: +# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/SuchNestedMuchWow# +# ^^^^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/AnotherNestedMuchWow# print(SuchNestedMuchWow().class_item) #^^^^ reference python-stdlib 3.13 builtins/print(). -# ^^^^^^^^^^^^^^^^^ reference local 0 +# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/SuchNestedMuchWow# print(AnotherNestedMuchWow().other_item) #^^^^ reference python-stdlib 3.13 builtins/print(). -# ^^^^^^^^^^^^^^^^^^^^ reference local 1 +# ^^^^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/AnotherNestedMuchWow# print(InitClass().init_item) #^^^^ reference python-stdlib 3.13 builtins/print(). -# ^^^^^^^^^ reference local 2 +# ^^^^^^^^^ reference snapshot-util 0.1 `foo.bar`/InitClass# diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py index 5e3100fd56..945f5ab8d7 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py @@ -1,8 +1,9 @@ -# < definition scip-python python snapshot-util 0.1 `src.long_importer`/__init__: +# < definition scip-python python snapshot-util 0.1 long_importer/__init__: import foo.bar.baz.mod +# ^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/__init__: print(foo.bar.baz.mod.SuchNestedMuchWow) #^^^^ reference python-stdlib 3.13 builtins/print(). -# ^^^ reference local 0 +# ^^^ reference snapshot-util 0.1 foo/__init__: diff --git a/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py index 7c044c5993..30df10eef8 100644 --- a/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py +++ b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py @@ -1,7 +1,11 @@ # < definition scip-python python snapshot-util 0.1 bar/__init__: from .foo import exported_function +# ^^^^ reference snapshot-util 0.1 foo/__init__: +# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 foo/exported_function(). from .foo import this_class +# ^^^^ reference snapshot-util 0.1 foo/__init__: +# ^^^^^^^^^^ reference snapshot-util 0.1 foo/this_class. if True: exported_function() diff --git a/pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py b/pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py new file mode 100644 index 0000000000..59d7b2fc08 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py @@ -0,0 +1,9 @@ +# < definition scip-python python snapshot-util 0.1 goofy/__init__: + +import requests +# ^^^^^^^^ reference requests 2.0.0 requests/__init__: + +print(requests.get("https://sourcegraph.com")) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^^^^^^ reference requests 2.0.0 requests/__init__: + diff --git a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py index 17c00f32f9..11cac86748 100644 --- a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py @@ -1,37 +1,37 @@ -# < definition scip-python python snapshot-util 0.1 `src.single_class`/__init__: +# < definition scip-python python snapshot-util 0.1 single_class/__init__: class ExampleClass: -# ^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass# +# ^^^^^^^^^^^^ definition snapshot-util 0.1 single_class/ExampleClass# a: int -# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#a. +# ^ definition snapshot-util 0.1 single_class/ExampleClass#a. # ^^^ reference python-stdlib 3.13 builtins/int# b: int -# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#b. +# ^ definition snapshot-util 0.1 single_class/ExampleClass#b. # ^^^ reference python-stdlib 3.13 builtins/int# c: str -# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#c. +# ^ definition snapshot-util 0.1 single_class/ExampleClass#c. # ^^^ reference python-stdlib 3.13 builtins/str# static_var = "Hello World" -# ^^^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#static_var. +# ^^^^^^^^^^ definition snapshot-util 0.1 single_class/ExampleClass#static_var. def __init__(self, a: int, b: int): -# ^^^^^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__(). -# ^^^^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) -# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(a) +# ^^^^^^^^ definition snapshot-util 0.1 single_class/ExampleClass#__init__(). +# ^^^^ definition snapshot-util 0.1 single_class/ExampleClass#__init__().(self) +# ^ definition snapshot-util 0.1 single_class/ExampleClass#__init__().(a) # ^^^ reference python-stdlib 3.13 builtins/int# -# ^ definition snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(b) +# ^ definition snapshot-util 0.1 single_class/ExampleClass#__init__().(b) # ^^^ reference python-stdlib 3.13 builtins/int# local_c = ", world!" # ^^^^^^^ definition local 0 self.a = a -# ^^^^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) -# ^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(a) +# ^^^^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(self) +# ^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(a) self.b = b -# ^^^^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) -# ^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(b) +# ^^^^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(self) +# ^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(b) self.c = "hello" + local_c -# ^^^^ reference snapshot-util 0.1 `src.single_class`/ExampleClass#__init__().(self) +# ^^^^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(self) # ^^^^^^^ reference local 0 diff --git a/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py index 87eb165a77..0c9746a0aa 100644 --- a/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py +++ b/pyrefly/lib/report/scip/snapshots/output/single_function/src/single_function.py @@ -1,35 +1,35 @@ -# < definition scip-python python snapshot-util 0.1 `src.single_function`/__init__: +# < definition scip-python python snapshot-util 0.1 single_function/__init__: def my_cool_function(a: str) -> str: -# ^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/my_cool_function(). -# ^ definition snapshot-util 0.1 `src.single_function`/my_cool_function().(a) +# ^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 single_function/my_cool_function(). +# ^ definition snapshot-util 0.1 single_function/my_cool_function().(a) # ^^^ reference python-stdlib 3.13 builtins/str# # ^^^ reference python-stdlib 3.13 builtins/str# x = ", world" # ^ definition local 0 return a + x -# ^ reference snapshot-util 0.1 `src.single_function`/my_cool_function().(a) +# ^ reference snapshot-util 0.1 single_function/my_cool_function().(a) # ^ reference local 0 def my_cool_function_2(a: str): -# ^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/my_cool_function_2(). -# ^ definition snapshot-util 0.1 `src.single_function`/my_cool_function_2().(a) +# ^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 single_function/my_cool_function_2(). +# ^ definition snapshot-util 0.1 single_function/my_cool_function_2().(a) # ^^^ reference python-stdlib 3.13 builtins/str# x = ", world" # ^ definition local 1 return (lambda y: a + x + y)("oh no") # ^ definition local 2 -# ^ reference snapshot-util 0.1 `src.single_function`/my_cool_function_2().(a) +# ^ reference snapshot-util 0.1 single_function/my_cool_function_2().(a) # ^ reference local 1 # ^ reference local 3 def next_level(): -# ^^^^^^^^^^ definition snapshot-util 0.1 `src.single_function`/next_level(). +# ^^^^^^^^^^ definition snapshot-util 0.1 single_function/next_level(). return my_cool_function -# ^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `src.single_function`/my_cool_function(). +# ^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 single_function/my_cool_function(). my_cool_function("hello") -#^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `src.single_function`/my_cool_function(). +#^^^^^^^^^^^^^^^ reference snapshot-util 0.1 single_function/my_cool_function(). next_level()(a = "wow") -#^^^^^^^^^ reference snapshot-util 0.1 `src.single_function`/next_level(). +#^^^^^^^^^ reference snapshot-util 0.1 single_function/next_level(). diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py index 583bfdaa0d..bb12da2e69 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py @@ -1,6 +1,8 @@ # < definition scip-python python snapshot-util 0.1 builtin_import_refs/__init__: from typing import Any +# ^^^^^^ reference python-stdlib 3.13 typing/__init__: +# ^^^ reference python-stdlib 3.13 typing/Any# print(Any) #^^^^ reference python-stdlib 3.13 builtins/print(). diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py index ec49e3c6a6..6afacaef8d 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py @@ -1,6 +1,8 @@ # < definition scip-python python snapshot-util 0.1 property_access/__init__: from typing import Sequence +# ^^^^^^ reference python-stdlib 3.13 typing/__init__: +# ^^^^^^^^ reference python-stdlib 3.13 typing/Sequence# class PropertyClass: # ^^^^^^^^^^^^^ definition snapshot-util 0.1 property_access/PropertyClass# diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py index cead9649f4..00ae3b84c1 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py @@ -1,6 +1,8 @@ # < definition scip-python python snapshot-util 0.1 vars_inside_scopes/__init__: from typing import List +# ^^^^^^ reference python-stdlib 3.13 typing/__init__: +# ^^^^ reference python-stdlib 3.13 typing/List. class X: # ^ definition snapshot-util 0.1 vars_inside_scopes/X# diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py index 02f69f4896..2bd42c52f3 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py @@ -1,6 +1,7 @@ # < definition scip-python python snapshot-util 0.1 walrus/__init__: import re +# ^^ reference python-stdlib 3.13 re/__init__: some_list = [1, 2, 3, 4, 5, 100, 1000] #^^^^^^^^ definition snapshot-util 0.1 walrus/some_list. diff --git a/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py b/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py index 95d33f3426..834950ffed 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py +++ b/pyrefly/lib/report/scip/snapshots/output/unresolved_import/unresolved.py @@ -1,6 +1,7 @@ # < definition scip-python python snapshot-util 0.1 unresolved/__init__: import this_is_not_real +# ^^^^^^^^^^^^^^^^ reference local 0 print(this_is_not_real.x) #^^^^ reference python-stdlib 3.13 builtins/print(). diff --git a/pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/METADATA b/pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/METADATA new file mode 100644 index 0000000000..a2f80ca6a8 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Metadata-Version: 2.1 +Name: requests +Version: 2.0.0 diff --git a/pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/RECORD b/pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/RECORD new file mode 100644 index 0000000000..3d91bfa356 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/site-packages/requests-2.0.0.dist-info/RECORD @@ -0,0 +1,3 @@ +requests/__init__.py,, +requests-2.0.0.dist-info/METADATA,, +requests-2.0.0.dist-info/RECORD,, diff --git a/pyrefly/lib/report/scip/snapshots/site-packages/requests/__init__.py b/pyrefly/lib/report/scip/snapshots/site-packages/requests/__init__.py new file mode 100644 index 0000000000..e69de29bb2 From c7b374571ed59dd8a4bac5268eab7c62989bd7e8 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sun, 18 Oct 2026 23:47:24 -0400 Subject: [PATCH 07/21] attribute references --- pyrefly/lib/report/scip.rs | 94 +++++++++++++------ .../snapshots/output/aliased_import/actual.py | 1 + .../output/class_nohint/class_nohint.py | 5 + .../output/enclosing_range_class/advanced.py | 1 + .../file_from_module_import/abc/file.py | 1 + .../file_from_module.py | 1 + .../output/nameparts_import/nameparts.py | 4 + .../output/nested_items/src/importer.py | 3 + .../output/nested_items/src/long_importer.py | 3 + .../snapshots/output/relative_import/bar.py | 1 + .../output/request_goofiness/goofy.py | 1 + .../output/single_class/src/single_class.py | 3 + .../output/unique/property_access.py | 1 + .../output/unique/vars_inside_scopes.py | 2 + .../scip/snapshots/output/unique/walrus.py | 2 + 15 files changed, 93 insertions(+), 30 deletions(-) diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 62a5df8b5a..b5fc18f335 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -184,14 +184,21 @@ impl Indexer<'_> { .unwrap_or(0); let mut unresolved: SmallMap = SmallMap::new(); let mut uses = Vec::new(); - ast.visit(&mut |x| collect_name_uses(x, &mut uses)); + ast.visit(&mut |x| collect_expr_uses(x, &mut uses)); ast.body.iter().for_each(|x| collect_imports(x, &mut uses)); + // The first `self.x = ...` of an attribute is its definition, not a use. + uses.retain(|x| !definitions.contains_key(&x.range)); uses.sort_by_key(|x| x.range.start()); for x in uses { - let symbol = self.reference(handle, &module, &x).unwrap_or_else(|| { - let id = first_unresolved + unresolved.len(); - format!("local {}", unresolved.entry(x.identifier.id).or_insert(id)) - }); + let symbol = match (self.reference(handle, &module, &x), x.kind) { + (Some(symbol), _) => symbol, + (None, UseKind::Name | UseKind::Import) => { + let id = first_unresolved + unresolved.len(); + format!("local {}", unresolved.entry(x.identifier.id).or_insert(id)) + } + // An unresolved attribute or keyword isn't a variable, so it gets no local. + (None, UseKind::Attribute | UseKind::Keyword) => continue, + }; occurrences.push(Occurrence { range: scip_range(&module, x.range), symbol, @@ -214,22 +221,27 @@ impl Indexer<'_> { resolve_call_dunders: false, ..Default::default() }; - let definition = if x.import { - // At the identifier's last character, a dotted module name resolves to the whole - // module rather than to its first component. - let position = x.identifier.range.end() - TextSize::from(1); - let definitions = self + let definition = match x.kind { + UseKind::Name => self .transaction - .find_definition(handle, position, preference) - .ok()?; - definitions.into_iter().next() - } else { - self.transaction .find_definition_for_name_use(handle, &x.identifier, preference) - .ok()? + .ok()?, + UseKind::Import | UseKind::Attribute | UseKind::Keyword => { + // At the identifier's last character, a dotted module name in an import resolves + // to the whole module rather than to its first component. + let position = x.identifier.range.end() - TextSize::from(1); + let definitions = self + .transaction + .find_definition(handle, position, preference) + .ok()?; + definitions.into_iter().next() + } }?; let package = self.package(&definition.module, handle.sys_info().dupe()); - if definition.metadata.symbol_kind() == Some(SymbolKind::Module) { + // A submodule accessed as an attribute resolves to the empty range at the module's start. + if definition.metadata.symbol_kind() == Some(SymbolKind::Module) + || definition.definition_range.is_empty() + { return Some(module_symbol(&package, definition.module.name())); } let symbols = self.definitions(&definition.module, handle.sys_info().dupe())?; @@ -345,22 +357,44 @@ struct Use { /// The range of the occurrence. For a relative module, it also covers the leading dots. range: TextRange, identifier: Identifier, - /// Whether the name is part of an import statement, rather than an expression. - import: bool, + kind: UseKind, } -/// Collects every name that is read or deleted. -fn collect_name_uses(x: &Expr, uses: &mut Vec) { - if let Expr::Name(name) = x - && name.ctx != ExprContext::Store - { +#[derive(Clone, Copy)] +enum UseKind { + /// A name in an expression. + Name, + /// A module or name in an import statement. + Import, + Attribute, + /// The name of a keyword argument in a call. + Keyword, +} + +/// Collects every name that is read or deleted, every attribute, and every keyword argument. +fn collect_expr_uses(x: &Expr, uses: &mut Vec) { + let mut push = |identifier: Identifier, kind| { uses.push(Use { - range: name.range, - identifier: Ast::expr_name_identifier(name.clone()), - import: false, - }); + range: identifier.range, + identifier, + kind, + }) + }; + match x { + Expr::Name(name) if name.ctx != ExprContext::Store => { + push(Ast::expr_name_identifier(name.clone()), UseKind::Name) + } + Expr::Attribute(attribute) => push(attribute.attr.clone(), UseKind::Attribute), + Expr::Call(call) => { + for keyword in call.arguments.keywords.iter() { + if let Some(arg) = &keyword.arg { + push(arg.clone(), UseKind::Keyword); + } + } + } + _ => {} } - x.recurse(&mut |x| collect_name_uses(x, uses)); + x.recurse(&mut |x| collect_expr_uses(x, uses)); } /// Collects the module and name identifiers of every import statement. @@ -369,7 +403,7 @@ fn collect_imports(x: &Stmt, uses: &mut Vec) { uses.push(Use { range, identifier: identifier.clone(), - import: true, + kind: UseKind::Import, }) }; let aliases = match x { diff --git a/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py index ed85a3d7d7..5eb646a533 100644 --- a/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py +++ b/pyrefly/lib/report/scip/snapshots/output/aliased_import/actual.py @@ -9,4 +9,5 @@ print(A.SOME_CONSTANT) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^ reference snapshot-util 0.1 aliased/__init__: +# ^^^^^^^^^^^^^ reference snapshot-util 0.1 aliased/SOME_CONSTANT. diff --git a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py index 49d0a6f315..c28bf4ac8e 100644 --- a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py +++ b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py @@ -19,9 +19,12 @@ def __init__(self, in_val): # ^^^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(in_val) self.x = self.x + 1 # ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) +# ^ reference snapshot-util 0.1 class_nohint/Example#x. # ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) +# ^ reference snapshot-util 0.1 class_nohint/Example#x. self.y = in_val # ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) +# ^ reference snapshot-util 0.1 class_nohint/Example#y. # ^^^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(in_val) def something(self): @@ -30,7 +33,9 @@ def something(self): print(self.x) # ^^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^ reference snapshot-util 0.1 class_nohint/Example#something().(self) +# ^ reference snapshot-util 0.1 class_nohint/Example#x. print(self.y) # ^^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^ reference snapshot-util 0.1 class_nohint/Example#something().(self) +# ^ reference snapshot-util 0.1 class_nohint/Example#y. diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py index 4b1a61955c..81fcd440be 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py @@ -35,4 +35,5 @@ def test(self) -> float: # ^^^^^ reference python-stdlib 3.13 builtins/float# return self.x # ^^^^ reference snapshot-util 0.1 advanced/Test#test().(self) +# ^ reference snapshot-util 0.1 advanced/Test#x. diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py index b9beb098c0..a3106d0b5f 100644 --- a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/abc/file.py @@ -7,4 +7,5 @@ print(nested_file.X) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: +# ^ reference snapshot-util 0.1 `xyz.nested_file`/X. diff --git a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py index 50feb26819..29387cbf9c 100644 --- a/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py +++ b/pyrefly/lib/report/scip/snapshots/output/file_from_module_import/file_from_module.py @@ -7,4 +7,5 @@ print(nested_file.X) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^^^^^^^^ reference snapshot-util 0.1 `xyz.nested_file`/__init__: +# ^ reference snapshot-util 0.1 `xyz.nested_file`/X. diff --git a/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py index 95a9753d3a..aa38c26333 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py +++ b/pyrefly/lib/report/scip/snapshots/output/nameparts_import/nameparts.py @@ -5,6 +5,10 @@ importlib.resources.read_text('pre_commit.resources', 'filename') #^^^^^^^^ reference python-stdlib 3.13 importlib/__init__: +# ^^^^^^^^^ reference python-stdlib 3.13 `importlib.resources`/__init__: +# ^^^^^^^^^ reference python-stdlib 3.13 `importlib.resources._functional`/read_text(). importlib.resources.read_text('pre_commit.resources', 'filename') #^^^^^^^^ reference python-stdlib 3.13 importlib/__init__: +# ^^^^^^^^^ reference python-stdlib 3.13 `importlib.resources`/__init__: +# ^^^^^^^^^ reference python-stdlib 3.13 `importlib.resources._functional`/read_text(). diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py index dd48d8967b..165962035b 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/importer.py @@ -11,10 +11,13 @@ print(SuchNestedMuchWow().class_item) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/SuchNestedMuchWow# +# ^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/SuchNestedMuchWow#class_item. print(AnotherNestedMuchWow().other_item) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/AnotherNestedMuchWow# +# ^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/AnotherNestedMuchWow#other_item. print(InitClass().init_item) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^^^^^^ reference snapshot-util 0.1 `foo.bar`/InitClass# +# ^^^^^^^^^ reference snapshot-util 0.1 `foo.bar`/InitClass#init_item. diff --git a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py index 945f5ab8d7..fc7411bf8d 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py +++ b/pyrefly/lib/report/scip/snapshots/output/nested_items/src/long_importer.py @@ -6,4 +6,7 @@ print(foo.bar.baz.mod.SuchNestedMuchWow) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^ reference snapshot-util 0.1 foo/__init__: +# ^^^ reference snapshot-util 0.1 `foo.bar`/__init__: +# ^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/__init__: +# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 `foo.bar.baz.mod`/SuchNestedMuchWow# diff --git a/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py index 30df10eef8..2dc2f5f07a 100644 --- a/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py +++ b/pyrefly/lib/report/scip/snapshots/output/relative_import/bar.py @@ -13,4 +13,5 @@ this_class.exported_function() # ^^^^^^^^^^ reference snapshot-util 0.1 foo/this_class. +# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 foo/MyClass#exported_function(). diff --git a/pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py b/pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py index 59d7b2fc08..89dac0fde5 100644 --- a/pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py +++ b/pyrefly/lib/report/scip/snapshots/output/request_goofiness/goofy.py @@ -6,4 +6,5 @@ print(requests.get("https://sourcegraph.com")) #^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^^^^^ reference requests 2.0.0 requests/__init__: +# ^^^ reference requests 2.0.0 `requests.api`/get(). diff --git a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py index 11cac86748..b9b89de9c0 100644 --- a/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/single_class/src/single_class.py @@ -27,11 +27,14 @@ def __init__(self, a: int, b: int): self.a = a # ^^^^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(self) +# ^ reference snapshot-util 0.1 single_class/ExampleClass#a. # ^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(a) self.b = b # ^^^^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(self) +# ^ reference snapshot-util 0.1 single_class/ExampleClass#b. # ^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(b) self.c = "hello" + local_c # ^^^^ reference snapshot-util 0.1 single_class/ExampleClass#__init__().(self) +# ^ reference snapshot-util 0.1 single_class/ExampleClass#c. # ^^^^^^^ reference local 0 diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py index 6afacaef8d..04770b64d3 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/property_access.py @@ -36,4 +36,5 @@ def nested(): print(x.prop_ref) # ^^^^^ reference python-stdlib 3.13 builtins/print(). # ^ reference local 0 +# ^^^^^^^^ reference snapshot-util 0.1 property_access/PropertyClass#prop_ref(). diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py index 00ae3b84c1..299a86fe88 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/vars_inside_scopes.py @@ -17,12 +17,14 @@ def my_func(self): for x in self.items: # ^ definition local 0 # ^^^^ reference snapshot-util 0.1 vars_inside_scopes/X#my_func().(self) +# ^^^^^ reference snapshot-util 0.1 vars_inside_scopes/X#items. y = x + 1 # ^ definition local 1 # ^ reference local 0 if 5 in self.items: # ^^^^ reference snapshot-util 0.1 vars_inside_scopes/X#my_func().(self) +# ^^^^^ reference snapshot-util 0.1 vars_inside_scopes/X#items. z = "oh ya" # ^ definition local 2 diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py index 2bd42c52f3..263e42a125 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/walrus.py @@ -22,11 +22,13 @@ if (match := re.search(pattern, text)) is not None: # ^^^^^ definition snapshot-util 0.1 walrus/match. # ^^ reference python-stdlib 3.13 re/__init__: +# ^^^^^^ reference python-stdlib 3.13 re/search(). # ^^^^^^^ reference snapshot-util 0.1 walrus/pattern. # ^^^^ reference snapshot-util 0.1 walrus/text. print("Found match:", match.group(0)) # ^^^^^ reference python-stdlib 3.13 builtins/print(). # ^^^^^ reference snapshot-util 0.1 walrus/match. +# ^^^^^ reference python-stdlib 3.13 re/Match#group(). # comprehensions From 5a4f60fa5dc82fdad4178cd7e6e37d2f1887201b Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Mon, 19 Oct 2026 00:19:46 -0400 Subject: [PATCH 08/21] enclosing ranges --- pyrefly/lib/report/scip.rs | 29 +++++++++++++++---- .../output/enclosing_range_class/advanced.py | 10 +++++++ .../output/enclosing_range_class/simple.py | 2 ++ .../enclosing_range_function/advanced.py | 6 ++++ .../output/enclosing_range_function/simple.py | 2 ++ 5 files changed, 44 insertions(+), 5 deletions(-) diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index b5fc18f335..c785513ab5 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -165,11 +165,20 @@ impl Indexer<'_> { symbol_roles: SymbolRole::Definition as i32, ..Default::default() }]; - occurrences.extend(definitions.iter().map(|(range, symbol)| Occurrence { - range: scip_range(&module, *range), - symbol: format_scip_symbol(self.project, symbol), - symbol_roles: SymbolRole::Definition as i32, - ..Default::default() + let mut enclosing_ranges = SmallMap::new(); + ast.body + .iter() + .for_each(|x| collect_enclosing_ranges(x, &mut enclosing_ranges)); + occurrences.extend(definitions.iter().map(|(range, symbol)| { + Occurrence { + range: scip_range(&module, *range), + symbol: format_scip_symbol(self.project, symbol), + symbol_roles: SymbolRole::Definition as i32, + enclosing_range: enclosing_ranges + .get(range) + .map_or_else(Vec::new, |x| scip_range(&module, *x)), + ..Default::default() + } })); // Names that resolve to no definition get one local per name, numbered after the @@ -397,6 +406,16 @@ fn collect_expr_uses(x: &Expr, uses: &mut Vec) { x.recurse(&mut |x| collect_expr_uses(x, uses)); } +/// Maps the name of every function and class to its whole definition, decorators included. +fn collect_enclosing_ranges(x: &Stmt, ranges: &mut SmallMap) { + match x { + Stmt::FunctionDef(x) => ranges.insert(x.name.range, x.range), + Stmt::ClassDef(x) => ranges.insert(x.name.range, x.range), + _ => None, + }; + x.recurse(&mut |x| collect_enclosing_ranges(x, ranges)); +} + /// Collects the module and name identifiers of every import statement. fn collect_imports(x: &Stmt, uses: &mut Vec) { let mut import = |range, identifier: &Identifier| { diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py index 81fcd440be..b99e3dc076 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/advanced.py @@ -1,9 +1,11 @@ # < definition scip-python python snapshot-util 0.1 advanced/__init__: # format-options: showRanges +# < start enclosing_range scip-python python snapshot-util 0.1 advanced/class_decorator(). def class_decorator(cls): # ^^^^^^^^^^^^^^^ definition snapshot-util 0.1 advanced/class_decorator(). # ^^^ definition snapshot-util 0.1 advanced/class_decorator().(cls) +# ⌄ start enclosing_range scip-python python snapshot-util 0.1 advanced/class_decorator().wrapper(). def wrapper(*args, **kwargs): # ^^^^^^^ definition snapshot-util 0.1 advanced/class_decorator().wrapper(). # ^^^^ definition snapshot-util 0.1 advanced/class_decorator().wrapper().(args) @@ -12,13 +14,17 @@ def wrapper(*args, **kwargs): # ^^^ reference snapshot-util 0.1 advanced/class_decorator().(cls) # ^^^^ reference snapshot-util 0.1 advanced/class_decorator().wrapper().(args) # ^^^^^^ reference snapshot-util 0.1 advanced/class_decorator().wrapper().(kwargs) +# ^ end enclosing_range scip-python python snapshot-util 0.1 advanced/class_decorator().wrapper(). return wrapper # ^^^^^^^ reference snapshot-util 0.1 advanced/class_decorator().wrapper(). +# < end enclosing_range scip-python python snapshot-util 0.1 advanced/class_decorator(). +# < start enclosing_range scip-python python snapshot-util 0.1 advanced/Test# @class_decorator #^^^^^^^^^^^^^^^ reference snapshot-util 0.1 advanced/class_decorator(). class Test: # ^^^^ definition snapshot-util 0.1 advanced/Test# +# ⌄ start enclosing_range scip-python python snapshot-util 0.1 advanced/Test#__init__(). def __init__(self, x: float): # ^^^^^^^^ definition snapshot-util 0.1 advanced/Test#__init__(). # ^^^^ definition snapshot-util 0.1 advanced/Test#__init__().(self) @@ -28,7 +34,9 @@ def __init__(self, x: float): # ^^^^ reference snapshot-util 0.1 advanced/Test#__init__().(self) # ^ definition snapshot-util 0.1 advanced/Test#x. # ^ reference snapshot-util 0.1 advanced/Test#__init__().(x) +# ^ end enclosing_range scip-python python snapshot-util 0.1 advanced/Test#__init__(). +# ⌄ start enclosing_range scip-python python snapshot-util 0.1 advanced/Test#test(). def test(self) -> float: # ^^^^ definition snapshot-util 0.1 advanced/Test#test(). # ^^^^ definition snapshot-util 0.1 advanced/Test#test().(self) @@ -36,4 +44,6 @@ def test(self) -> float: return self.x # ^^^^ reference snapshot-util 0.1 advanced/Test#test().(self) # ^ reference snapshot-util 0.1 advanced/Test#x. +# ^ end enclosing_range scip-python python snapshot-util 0.1 advanced/Test#test(). +# < end enclosing_range scip-python python snapshot-util 0.1 advanced/Test# diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py index c9c70ea6fd..71a71c002a 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_class/simple.py @@ -1,7 +1,9 @@ # < definition scip-python python snapshot-util 0.1 simple/__init__: # format-options: showRanges +# < start enclosing_range scip-python python snapshot-util 0.1 simple/Test# class Test(): # ^^^^ definition snapshot-util 0.1 simple/Test# pass +# < end enclosing_range scip-python python snapshot-util 0.1 simple/Test# diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py index 36d99e0482..74f40c6e35 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/advanced.py @@ -1,9 +1,11 @@ # < definition scip-python python snapshot-util 0.1 advanced/__init__: # format-options: showRanges +# < start enclosing_range scip-python python snapshot-util 0.1 advanced/decorator(). def decorator(func): # ^^^^^^^^^ definition snapshot-util 0.1 advanced/decorator(). # ^^^^ definition snapshot-util 0.1 advanced/decorator().(func) +# ⌄ start enclosing_range scip-python python snapshot-util 0.1 advanced/decorator().wrapper(). def wrapper(*args, **kwargs): # ^^^^^^^ definition snapshot-util 0.1 advanced/decorator().wrapper(). # ^^^^ definition snapshot-util 0.1 advanced/decorator().wrapper().(args) @@ -12,9 +14,12 @@ def wrapper(*args, **kwargs): # ^^^^ reference snapshot-util 0.1 advanced/decorator().(func) # ^^^^ reference snapshot-util 0.1 advanced/decorator().wrapper().(args) # ^^^^^^ reference snapshot-util 0.1 advanced/decorator().wrapper().(kwargs) +# ^ end enclosing_range scip-python python snapshot-util 0.1 advanced/decorator().wrapper(). return wrapper # ^^^^^^^ reference snapshot-util 0.1 advanced/decorator().wrapper(). +# < end enclosing_range scip-python python snapshot-util 0.1 advanced/decorator(). +# < start enclosing_range scip-python python snapshot-util 0.1 advanced/func(). @decorator #^^^^^^^^^ reference snapshot-util 0.1 advanced/decorator(). def func(x: float) -> float: @@ -28,4 +33,5 @@ def func(x: float) -> float: return test # ^^^^ reference local 0 +# < end enclosing_range scip-python python snapshot-util 0.1 advanced/func(). diff --git a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py index f845d5e004..fe80f65e24 100644 --- a/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py +++ b/pyrefly/lib/report/scip/snapshots/output/enclosing_range_function/simple.py @@ -1,9 +1,11 @@ # < definition scip-python python snapshot-util 0.1 simple/__init__: # format-options: showRanges +# < start enclosing_range scip-python python snapshot-util 0.1 simple/simple(). def simple(x): # ^^^^^^ definition snapshot-util 0.1 simple/simple(). # ^ definition snapshot-util 0.1 simple/simple().(x) return x # ^ reference snapshot-util 0.1 simple/simple().(x) +# < end enclosing_range scip-python python snapshot-util 0.1 simple/simple(). From 36a23eda9441a392b42c84073ca5cd3112a02a9a Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Wed, 21 Oct 2026 20:34:42 -0400 Subject: [PATCH 09/21] override relationships --- pyrefly/lib/report/scip.rs | 118 +++++++++++++++++- .../scip/snapshots/output/graph_1278/mwe.py | 3 + .../output/unique/grandparent_impl.py | 3 + .../snapshots/output/unique/inherits_class.py | 3 + .../output/unique/multiinherits_test.py | 5 + 5 files changed, 127 insertions(+), 5 deletions(-) diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index c785513ab5..584944b897 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -31,6 +31,7 @@ use ruff_python_ast::Expr; use ruff_python_ast::ExprContext; use ruff_python_ast::Identifier; use ruff_python_ast::Stmt; +use ruff_python_ast::StmtClassDef; use ruff_python_ast::name::Name; use ruff_text_size::Ranged; use ruff_text_size::TextRange; @@ -42,7 +43,9 @@ use scip::types::Index; use scip::types::Metadata; use scip::types::Occurrence; use scip::types::PositionEncoding; +use scip::types::Relationship; use scip::types::Symbol; +use scip::types::SymbolInformation; use scip::types::SymbolRole; use scip::types::TextEncoding; use scip::types::ToolInfo; @@ -53,6 +56,7 @@ use crate::config::config::ConfigFile; use crate::report::scip::symbols::ScipSymbol; use crate::report::scip::symbols::definitions; use crate::report::scip::symbols::descriptor; +use crate::state::lsp::FindDefinitionItemWithDocstring; use crate::state::lsp::FindPreference; use crate::state::state::Transaction; @@ -214,22 +218,99 @@ impl Indexer<'_> { ..Default::default() }); } + let mut classes = Vec::new(); + ast.body + .iter() + .for_each(|x| collect_classes(x, &mut classes)); + let mut symbols = Vec::new(); + for class in classes { + for (range, implemented) in self.implementations(handle, &module, class) { + let Some(symbol) = definitions.get(&range) else { + continue; + }; + symbols.push(SymbolInformation { + symbol: format_scip_symbol(self.project, symbol), + relationships: implemented + .into_iter() + .map(|symbol| Relationship { + symbol, + is_implementation: true, + ..Default::default() + }) + .collect(), + ..Default::default() + }); + } + } Some(Document { language: "python".to_owned(), relative_path: path_to_unix_string(relative_path), position_encoding: PositionEncoding::UTF32CodeUnitOffsetFromLineStart.into(), occurrences, + symbols, ..Default::default() }) } + /// The symbols that a class and its methods implement, keyed by the range of the defining + /// name. A class implements each of its bases, and a method implements the first method of + /// the same name found through the bases, in order. A method whose name starts with `__` + /// implements nothing. + fn implementations( + &mut self, + handle: &Handle, + module: &Module, + class: &StmtClassDef, + ) -> Vec<(TextRange, Vec)> { + let bases: Vec = class + .bases() + .iter() + .filter_map(|base| { + let base = match base { + Expr::Subscript(x) => &*x.value, + x => x, + }; + let (identifier, kind) = match base { + Expr::Name(x) => (Ast::expr_name_identifier(x.clone()), UseKind::Name), + Expr::Attribute(x) => (x.attr.clone(), UseKind::Attribute), + _ => return None, + }; + Some(Use { + range: base.range(), + identifier, + kind, + }) + }) + .collect(); + let class_bases: Vec<_> = bases + .iter() + .filter_map(|base| self.reference(handle, module, base)) + .collect(); + let mut implementations = vec![(class.name.range, class_bases)]; + for x in &class.body { + let Stmt::FunctionDef(method) = x else { + continue; + }; + let name = &method.name.id; + if name.starts_with("__") { + continue; + } + let implemented = bases.iter().find_map(|base| { + let definitions = self + .transaction + .find_definition_for_attribute(handle, base.range, name, preference()) + .ok()?; + self.symbol(handle, module, definitions.into_iter().next()?) + }); + implementations.push((method.name.range, implemented.into_iter().collect())); + } + implementations.retain(|(_, implemented)| !implemented.is_empty()); + implementations + } + /// The symbol a use refers to, or `None` if it has no definition we can name. fn reference(&mut self, handle: &Handle, module: &Module, x: &Use) -> Option { - let preference = FindPreference { - // A call like `C()` refers to the class `C`, not to `C.__init__`. - resolve_call_dunders: false, - ..Default::default() - }; + let preference = preference(); let definition = match x.kind { UseKind::Name => self .transaction @@ -246,6 +327,17 @@ impl Indexer<'_> { definitions.into_iter().next() } }?; + self.symbol(handle, module, definition) + } + + /// The symbol of a definition, or `None` if it is a local of another module or has no + /// definition we can name. + fn symbol( + &mut self, + handle: &Handle, + module: &Module, + definition: FindDefinitionItemWithDocstring, + ) -> Option { let package = self.package(&definition.module, handle.sys_info().dupe()); // A submodule accessed as an attribute resolves to the empty range at the module's start. if definition.metadata.symbol_kind() == Some(SymbolKind::Module) @@ -406,6 +498,22 @@ fn collect_expr_uses(x: &Expr, uses: &mut Vec) { x.recurse(&mut |x| collect_expr_uses(x, uses)); } +fn preference() -> FindPreference { + FindPreference { + // A call like `C()` refers to the class `C`, not to `C.__init__`. + resolve_call_dunders: false, + ..Default::default() + } +} + +/// Collects every class definition, including nested ones. +fn collect_classes<'a>(x: &'a Stmt, classes: &mut Vec<&'a StmtClassDef>) { + if let Stmt::ClassDef(x) = x { + classes.push(x); + } + x.recurse(&mut |x| collect_classes(x, classes)); +} + /// Maps the name of every function and class to its whole definition, decorators included. fn collect_enclosing_ranges(x: &Stmt, ranges: &mut SmallMap) { match x { diff --git a/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py index 0e4078b562..b7c9b702c2 100644 --- a/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py +++ b/pyrefly/lib/report/scip/snapshots/output/graph_1278/mwe.py @@ -17,6 +17,7 @@ class X(Generic[_T_co]): # ^ definition snapshot-util 0.1 mwe/X# +# relationship implementation scip-python python python-stdlib 3.13 typing/Generic. # ^^^^^^^ reference python-stdlib 3.13 typing/Generic. # ^^^^^ reference snapshot-util 0.1 mwe/_T_co. pass @@ -50,9 +51,11 @@ def noop(): class FooImpl(Foo): # ^^^^^^^ definition snapshot-util 0.1 mwe/FooImpl# +# relationship implementation scip-python python snapshot-util 0.1 mwe/Foo# # ^^^ reference snapshot-util 0.1 mwe/Foo# def foo(self): # ^^^ definition snapshot-util 0.1 mwe/FooImpl#foo(). +# relationship implementation scip-python python snapshot-util 0.1 mwe/Foo#foo(). # ^^^^ definition snapshot-util 0.1 mwe/FooImpl#foo().(self) return noop() # ^^^^ reference snapshot-util 0.1 mwe/noop(). diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py index 7c8fd05793..ec0f111d19 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/grandparent_impl.py @@ -10,14 +10,17 @@ def grandparent(self) -> bool: class B(A): # ^ definition snapshot-util 0.1 grandparent_impl/B# +# relationship implementation scip-python python snapshot-util 0.1 grandparent_impl/A# # ^ reference snapshot-util 0.1 grandparent_impl/A# ... class C(B): # ^ definition snapshot-util 0.1 grandparent_impl/C# +# relationship implementation scip-python python snapshot-util 0.1 grandparent_impl/B# # ^ reference snapshot-util 0.1 grandparent_impl/B# def grandparent(self) -> bool: # ^^^^^^^^^^^ definition snapshot-util 0.1 grandparent_impl/C#grandparent(). +# relationship implementation scip-python python snapshot-util 0.1 grandparent_impl/A#grandparent(). # ^^^^ definition snapshot-util 0.1 grandparent_impl/C#grandparent().(self) # ^^^^ reference python-stdlib 3.13 builtins/bool# return False diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py index f6394ab6db..7edf578305 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py @@ -18,15 +18,18 @@ def matched_despite_different_type(self, x: int): class B(A): # ^ definition snapshot-util 0.1 inherits_class/B# +# relationship implementation scip-python python snapshot-util 0.1 inherits_class/A# # ^ reference snapshot-util 0.1 inherits_class/A# def x(self) -> int: # ^ definition snapshot-util 0.1 inherits_class/B#x(). +# relationship implementation scip-python python snapshot-util 0.1 inherits_class/A#x(). # ^^^^ definition snapshot-util 0.1 inherits_class/B#x().(self) # ^^^ reference python-stdlib 3.13 builtins/int# return 5 def matched_despite_different_type(self, x: int, y: int): # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type(). +# relationship implementation scip-python python snapshot-util 0.1 inherits_class/A#matched_despite_different_type(). # ^^^^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(self) # ^ definition snapshot-util 0.1 inherits_class/B#matched_despite_different_type().(x) # ^^^ reference python-stdlib 3.13 builtins/int# diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py index fd38f83aa7..7b870fb1ed 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/multiinherits_test.py @@ -29,16 +29,20 @@ def shared(self) -> bool: class Multi(Left, Right): # ^^^^^ definition snapshot-util 0.1 multiinherits_test/Multi# +# relationship implementation scip-python python snapshot-util 0.1 multiinherits_test/Left# +# relationship implementation scip-python python snapshot-util 0.1 multiinherits_test/Right# # ^^^^ reference snapshot-util 0.1 multiinherits_test/Left# # ^^^^^ reference snapshot-util 0.1 multiinherits_test/Right# def one(self) -> int: # ^^^ definition snapshot-util 0.1 multiinherits_test/Multi#one(). +# relationship implementation scip-python python snapshot-util 0.1 multiinherits_test/Left#one(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#one().(self) # ^^^ reference python-stdlib 3.13 builtins/int# return 1 def two(self): # ^^^ definition snapshot-util 0.1 multiinherits_test/Multi#two(). +# relationship implementation scip-python python snapshot-util 0.1 multiinherits_test/Right#two(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#two().(self) return 2 @@ -49,6 +53,7 @@ def three(self): def shared(self) -> bool: # ^^^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#shared(). +# relationship implementation scip-python python snapshot-util 0.1 multiinherits_test/Left#shared(). # ^^^^ definition snapshot-util 0.1 multiinherits_test/Multi#shared().(self) # ^^^^ reference python-stdlib 3.13 builtins/bool# return True From c541ae63eb54994718af6f82b798c2c33e57a7b9 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Wed, 21 Oct 2026 23:52:46 -0400 Subject: [PATCH 10/21] docstrings --- pyrefly/lib/binding/class.rs | 5 +- pyrefly/lib/lsp/non_wasm/server.rs | 1 + pyrefly/lib/lsp/wasm/hover.rs | 31 +++- pyrefly/lib/report/scip.rs | 173 +++++++++++++++--- .../output/builtin_imports/builtin_imports.py | 36 ++++ .../output/class_nohint/class_nohint.py | 78 ++++++++ .../output/unique/field_docstring.py | 38 ++++ .../output/unique/inferred_field_docstring.py | 48 +++++ .../output/unique/module_docstring.py | 8 + pyrefly/lib/test/lsp/hover.rs | 39 ++++ 10 files changed, 426 insertions(+), 31 deletions(-) diff --git a/pyrefly/lib/binding/class.rs b/pyrefly/lib/binding/class.rs index c5b4d74bbf..e15b9c3be1 100644 --- a/pyrefly/lib/binding/class.rs +++ b/pyrefly/lib/binding/class.rs @@ -252,7 +252,7 @@ impl<'a> BindingsBuilder<'a> { let mut pydantic_config_dict = PydanticConfigDict::default(); let docstring_range = Docstring::range_from_stmts(x.body.as_slice()); let body = mem::take(&mut x.body); - let field_docstrings = self.extract_field_docstrings(&body); + let field_docstrings = Self::extract_field_docstrings(&body); let pydantic_before_validator_fields = self.extract_field_validator_fields(&body); let mut attrs_decorators = AttrsDecoratorMethods::default(); collect_attrs_decorator_methods(&body, &mut attrs_decorators); @@ -612,8 +612,7 @@ impl<'a> BindingsBuilder<'a> { } /// Extracts docstrings for each field, mapping the field's range to the docstring's range. - fn extract_field_docstrings( - &self, + pub fn extract_field_docstrings( body: &[ruff_python_ast::Stmt], ) -> SmallMap { use ruff_python_ast::Expr; diff --git a/pyrefly/lib/lsp/non_wasm/server.rs b/pyrefly/lib/lsp/non_wasm/server.rs index 4fcbd52e3a..cf29f5c077 100644 --- a/pyrefly/lib/lsp/non_wasm/server.rs +++ b/pyrefly/lib/lsp/non_wasm/server.rs @@ -5812,6 +5812,7 @@ impl Server { HoverOptions { show_go_to_links, verbosity_level, + field_docstring_at_definition: false, }, )) } diff --git a/pyrefly/lib/lsp/wasm/hover.rs b/pyrefly/lib/lsp/wasm/hover.rs index c4ce55b0ef..cb88eb44a2 100644 --- a/pyrefly/lib/lsp/wasm/hover.rs +++ b/pyrefly/lib/lsp/wasm/hover.rs @@ -53,6 +53,7 @@ use vec1::Vec1; use crate::alt::answers_solver::AnswersSolver; use crate::binding::binding::Key; +use crate::binding::bindings::BindingsBuilder; use crate::error::error::Error; use crate::lsp::module_helpers::collect_symbol_def_paths; use crate::lsp::module_helpers::to_real_path; @@ -100,6 +101,9 @@ pub struct HoverResult { pub struct HoverOptions { pub show_go_to_links: bool, pub verbosity_level: usize, + /// Whether hovering the definition of a class field shows the docstring that follows it. + /// Hovering a use of the field always does. + pub field_docstring_at_definition: bool, } /// Hover qualifies the module of a value's type, but not of a signature, where a @@ -909,6 +913,24 @@ fn resolve_hover_parameter_doc( None } +/// The docstring after a class field's definition, if `position` is on the field's name there. +fn field_docstring_at_definition( + ast: &ModModule, + module: Module, + position: TextSize, +) -> Option { + let nodes = Ast::locate_node(ast, position); + let AnyNodeRef::ExprName(name) = nodes.first()? else { + return None; + }; + let class = nodes.iter().find_map(|node| match node { + AnyNodeRef::StmtClassDef(class) => Some(*class), + _ => None, + })?; + let range = *BindingsBuilder::extract_field_docstrings(&class.body).get(&name.range)?; + Some(Docstring(range, module)) +} + pub fn get_hover( transaction: &Transaction<'_>, handle: &Handle, @@ -922,6 +944,7 @@ pub fn get_hover( HoverOptions { show_go_to_links, verbosity_level: 0, + field_docstring_at_definition: false, }, ) .map(|result| result.hover) @@ -1037,7 +1060,13 @@ pub fn get_hover_with_verbosity( .map(|range| Docstring(range, item.module)) }) .filter(|docstring| !docstring.resolve().trim().is_empty()) - .or(fallback_docstring); + .or(fallback_docstring) + .or_else(|| { + options + .field_docstring_at_definition + .then(|| field_docstring_at_definition(ast.as_deref()?, module_info?, position)) + .flatten() + }); let name = name.or_else(|| identifier_text_at(transaction, handle, position)); diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 584944b897..675bc76f71 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -12,9 +12,12 @@ use std::path::PathBuf; use std::sync::Arc; use dupe::Dupe; +use lsp_types::Contents; +use lsp_types::Hover; use lsp_types::Uri; use pyrefly_build::handle::Handle; use pyrefly_python::ast::Ast; +use pyrefly_python::docstring::Docstring; use pyrefly_python::module::Module; use pyrefly_python::module_name::ModuleName; use pyrefly_python::module_path::ModulePath; @@ -53,6 +56,9 @@ use scip::types::descriptor::Suffix; use starlark_map::small_map::SmallMap; use crate::config::config::ConfigFile; +use crate::lsp::wasm::hover::HoverOptions; +use crate::lsp::wasm::hover::HoverResult; +use crate::lsp::wasm::hover::get_hover_with_verbosity; use crate::report::scip::symbols::ScipSymbol; use crate::report::scip::symbols::definitions; use crate::report::scip::symbols::descriptor; @@ -117,12 +123,24 @@ pub fn scip( project, definitions: HashMap::new(), distributions: HashMap::new(), + external_symbols: HashMap::new(), }; let mut documents: Vec = handles .iter() .filter_map(|handle| indexer.document(handle)) .collect(); documents.sort_by(|a, b| a.relative_path.cmp(&b.relative_path)); + let mut external_symbols: Vec = indexer + .external_symbols + .into_iter() + .filter(|(_, documentation)| !documentation.is_empty()) + .map(|(symbol, documentation)| SymbolInformation { + symbol, + documentation, + ..Default::default() + }) + .collect(); + external_symbols.sort_by(|a, b| a.symbol.cmp(&b.symbol)); Index { metadata: Some(Metadata { project_root: Uri::from_directory_path(&project_root) @@ -139,6 +157,7 @@ pub fn scip( }) .into(), documents, + external_symbols, ..Default::default() } } @@ -151,6 +170,8 @@ struct Indexer<'a> { definitions: HashMap<(ModuleName, ModulePath), Arc>>, /// The installed distributions of every site-packages directory seen so far. distributions: HashMap>, + /// The documentation of every symbol defined outside the project and referenced so far. + external_symbols: HashMap>, } impl Indexer<'_> { @@ -204,7 +225,13 @@ impl Indexer<'_> { uses.sort_by_key(|x| x.range.start()); for x in uses { let symbol = match (self.reference(handle, &module, &x), x.kind) { - (Some(symbol), _) => symbol, + (Some((symbol, external)), _) => { + if external && !self.external_symbols.contains_key(&symbol) { + let documentation = self.documentation(handle, use_position(&x)); + self.external_symbols.insert(symbol.clone(), documentation); + } + symbol + } (None, UseKind::Name | UseKind::Import) => { let id = first_unresolved + unresolved.len(); format!("local {}", unresolved.entry(x.identifier.id).or_insert(id)) @@ -218,30 +245,61 @@ impl Indexer<'_> { ..Default::default() }); } + // A symbol defined more than once is documented at its first definition. + let mut symbols: SmallMap = SmallMap::new(); + let mut module_documentation = format!("```python\n(module) {}\n```", module.name()); + if let Some(range) = Docstring::range_from_stmts(&ast.body) { + module_documentation.push_str("\n---\n"); + module_documentation.push_str(Docstring(range, module.dupe()).resolve().trim()); + } + let module_symbol = module_symbol(self.project, module.name()); + symbols.insert( + module_symbol.clone(), + SymbolInformation { + symbol: module_symbol, + documentation: vec![module_documentation], + ..Default::default() + }, + ); + for (range, symbol) in definitions.iter() { + let symbol = format_scip_symbol(self.project, symbol); + if !symbols.contains_key(&symbol) { + let documentation = self.documentation(handle, range.start()); + symbols.insert( + symbol.clone(), + SymbolInformation { + symbol, + documentation, + ..Default::default() + }, + ); + } + } let mut classes = Vec::new(); ast.body .iter() .for_each(|x| collect_classes(x, &mut classes)); - let mut symbols = Vec::new(); for class in classes { for (range, implemented) in self.implementations(handle, &module, class) { let Some(symbol) = definitions.get(&range) else { continue; }; - symbols.push(SymbolInformation { - symbol: format_scip_symbol(self.project, symbol), - relationships: implemented + if let Some(info) = symbols.get_mut(&format_scip_symbol(self.project, symbol)) { + info.relationships = implemented .into_iter() .map(|symbol| Relationship { symbol, is_implementation: true, ..Default::default() }) - .collect(), - ..Default::default() - }); + .collect(); + } } } + let symbols = symbols + .into_values() + .filter(|info| !info.documentation.is_empty() || !info.relationships.is_empty()) + .collect(); Some(Document { language: "python".to_owned(), relative_path: path_to_unix_string(relative_path), @@ -284,7 +342,10 @@ impl Indexer<'_> { .collect(); let class_bases: Vec<_> = bases .iter() - .filter_map(|base| self.reference(handle, module, base)) + .filter_map(|base| { + self.reference(handle, module, base) + .map(|(symbol, _)| symbol) + }) .collect(); let mut implementations = vec![(class.name.range, class_bases)]; for x in &class.body { @@ -300,7 +361,9 @@ impl Indexer<'_> { .transaction .find_definition_for_attribute(handle, base.range, name, preference()) .ok()?; - self.symbol(handle, module, definitions.into_iter().next()?) + let definition = definitions.into_iter().next()?; + self.symbol(handle, module, definition) + .map(|(symbol, _)| symbol) }); implementations.push((method.name.range, implemented.into_iter().collect())); } @@ -308,8 +371,29 @@ impl Indexer<'_> { implementations } - /// The symbol a use refers to, or `None` if it has no definition we can name. - fn reference(&mut self, handle: &Handle, module: &Module, x: &Use) -> Option { + /// The hover text at a position, as SCIP documentation. + fn documentation(&self, handle: &Handle, position: TextSize) -> Vec { + let options = HoverOptions { + show_go_to_links: false, + verbosity_level: 0, + field_docstring_at_definition: true, + }; + match get_hover_with_verbosity(self.transaction, handle, position, options) { + Some(HoverResult { + hover: + Hover { + contents: Contents::MarkupContent(markup), + .. + }, + .. + }) => vec![markup.value], + _ => Vec::new(), + } + } + + /// The symbol a use refers to, and whether it is defined outside the project, or `None` if + /// it has no definition we can name. + fn reference(&mut self, handle: &Handle, module: &Module, x: &Use) -> Option<(String, bool)> { let preference = preference(); let definition = match x.kind { UseKind::Name => self @@ -317,12 +401,9 @@ impl Indexer<'_> { .find_definition_for_name_use(handle, &x.identifier, preference) .ok()?, UseKind::Import | UseKind::Attribute | UseKind::Keyword => { - // At the identifier's last character, a dotted module name in an import resolves - // to the whole module rather than to its first component. - let position = x.identifier.range.end() - TextSize::from(1); let definitions = self .transaction - .find_definition(handle, position, preference) + .find_definition(handle, use_position(x), preference) .ok()?; definitions.into_iter().next() } @@ -330,26 +411,33 @@ impl Indexer<'_> { self.symbol(handle, module, definition) } - /// The symbol of a definition, or `None` if it is a local of another module or has no - /// definition we can name. + /// The symbol of a definition, and whether it is defined outside the project, or `None` if + /// it is a local of another module or has no definition we can name. fn symbol( &mut self, handle: &Handle, module: &Module, definition: FindDefinitionItemWithDocstring, - ) -> Option { + ) -> Option<(String, bool)> { + let external = !definition + .module + .path() + .as_path() + .starts_with(self.project_root); let package = self.package(&definition.module, handle.sys_info().dupe()); // A submodule accessed as an attribute resolves to the empty range at the module's start. if definition.metadata.symbol_kind() == Some(SymbolKind::Module) || definition.definition_range.is_empty() { - return Some(module_symbol(&package, definition.module.name())); + return Some((module_symbol(&package, definition.module.name()), external)); } let symbols = self.definitions(&definition.module, handle.sys_info().dupe())?; match symbols.get(&definition.definition_range)? { - ScipSymbol::Global(descriptors) => Some(global_symbol(&package, descriptors.clone())), + ScipSymbol::Global(descriptors) => { + Some((global_symbol(&package, descriptors.clone()), external)) + } ScipSymbol::Local(id) => { - (definition.module.path() == module.path()).then(|| format!("local {id}")) + (definition.module.path() == module.path()).then(|| (format!("local {id}"), false)) } } } @@ -498,6 +586,12 @@ fn collect_expr_uses(x: &Expr, uses: &mut Vec) { x.recurse(&mut |x| collect_expr_uses(x, uses)); } +/// The position at which a use is resolved. At the identifier's last character, a dotted module +/// name in an import resolves to the whole module rather than to its first component. +fn use_position(x: &Use) -> TextSize { + x.identifier.range.end() - TextSize::from(1) +} + fn preference() -> FindPreference { FindPreference { // A call like `C()` refers to the class `C`, not to `C.__init__`. @@ -607,6 +701,8 @@ fn module_symbol(package: &Package, module: ModuleName) -> String { #[cfg(test)] mod tests { + use protobuf::Message as _; + use super::*; use crate::report::scip::snapshot::format_snapshot; use crate::state::require::Require; @@ -634,11 +730,9 @@ mod tests { Package::project(&input_dir, field("name"), field("version")).unwrap() } - fn assert_snapshot(case: &str) { + fn index(case: &str) -> Index { let snapshots = snapshots_dir(); let input_dir = snapshots.join("input").join(case); - let output_dir = snapshots.join("output").join(case); - let mut env = TestEnv::new(); let mut handles = Vec::new(); for path in glob::glob(&format!("{}/**/*.py", input_dir.display())).unwrap() { @@ -660,14 +754,20 @@ mod tests { .with_site_package_paths(vec![snapshots.join("site-packages")]) .with_default_require_level(Require::Everything) .to_state(); - let index = scip( + scip( &state.transaction(), &handles, &input_dir, &project_package(&snapshots, case), "test", - ); + ) + } + fn assert_snapshot(case: &str) { + let snapshots = snapshots_dir(); + let input_dir = snapshots.join("input").join(case); + let output_dir = snapshots.join("output").join(case); + let index = index(case); let update = std::env::var("SCIP_UPDATE_SNAPSHOTS").is_ok(); for doc in &index.documents { let input = fs_anyhow::read_to_string(&input_dir.join(&doc.relative_path)).unwrap(); @@ -693,6 +793,25 @@ mod tests { } } + #[test] + fn external_symbols_are_deduplicated() { + // `print` and `int` are each referenced more than once. + let bytes = index("class_nohint").write_to_bytes().unwrap(); + let symbols: Vec = Index::parse_from_bytes(&bytes) + .unwrap() + .external_symbols + .into_iter() + .map(|x| x.symbol) + .collect(); + assert_eq!( + symbols, + [ + "scip-python python python-stdlib 3.13 builtins/int#", + "scip-python python python-stdlib 3.13 builtins/print().", + ] + ); + } + macro_rules! snapshot_tests { ($($case:ident),* $(,)?) => { $( diff --git a/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py index 095c0589aa..62010e5c43 100644 --- a/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py +++ b/pyrefly/lib/report/scip/snapshots/output/builtin_imports/builtin_imports.py @@ -1,16 +1,52 @@ # < definition scip-python python snapshot-util 0.1 builtin_imports/__init__: +#documentation ```python +# > (module) builtin_imports +# > ``` # format-options: showDocs import re # ^^ reference python-stdlib 3.13 re/__init__: +# external documentation ```python +# > (module) re: Module[re] +# > ``` from typing import Callable, Optional # ^^^^^^ reference python-stdlib 3.13 typing/__init__: +# external documentation ```python +# > (module) typing: Module[typing] +# > ``` # ^^^^^^^^ reference python-stdlib 3.13 typing/Callable. +# external documentation ```python +# > (class) Callable: type[Callable] +# > ``` # ^^^^^^^^ reference python-stdlib 3.13 typing/Optional. +# external documentation ```python +# > (class) Optional: type[Optional] +# > ``` print(re, Callable, Optional) #^^^^ reference python-stdlib 3.13 builtins/print(). +#external documentation ```python +# > (function) print: def print( +# > *values: object, +# > *, +# > sep : str | None =... +# > end : str | None =... +# > file : SupportsWrite[str] | None =... +# > flush : Literal[False] =... +# > ) -> None: ... +# > ``` +# > --- +# > Prints the values to a stream, or to sys... +# > +# > sep +# >   string inserted between valu... +# > end +# >   string appended after the la... +# > file +# >   a file-like object (stream);... +# > flush +# >   whether to forcibly flush th... # ^^ reference python-stdlib 3.13 re/__init__: # ^^^^^^^^ reference python-stdlib 3.13 typing/Callable. # ^^^^^^^^ reference python-stdlib 3.13 typing/Optional. diff --git a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py index c28bf4ac8e..e9570af43d 100644 --- a/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py +++ b/pyrefly/lib/report/scip/snapshots/output/class_nohint/class_nohint.py @@ -1,21 +1,72 @@ # < definition scip-python python snapshot-util 0.1 class_nohint/__init__: +#documentation ```python +# > (module) class_nohint +# > ``` # format-options: showDocs class Example: # ^^^^^^^ definition snapshot-util 0.1 class_nohint/Example# +# documentation ```python +# > (class) Example: type[Example] +# > ``` # Note, only y has a type hint y: int # ^ definition snapshot-util 0.1 class_nohint/Example#y. +# documentation ```python +# > (variable) y: int +# > ``` # ^^^ reference python-stdlib 3.13 builtins/int# +# external documentation ```python +# > (class) int: +# > @overload +# > def __new__(x: ConvertibleToInt = 0, /) ... +# > def __new__( +# > x: bytearray | bytes | str, +# > /, +# > base: SupportsIndex +# > ) -> int: ... +# > ``` +# > --- +# > int([x]) -> integer +# > int(x, base=10) -> integer +# > +# > Convert a number or string to an integer... +# > are given. If x is a number, return x._... +# > numbers, this truncates towards zero. +# > +# > If x is not a number or if base is given... +# > bytes, or bytearray instance representin... +# > given base. The literal can be preceded... +# > by whitespace. The base defaults to 10.... +# > Base 0 means to interpret the base from ... +# > ```python +# > >>> int('0b100', base=0) +# > ``` +# > 4 def __init__(self, in_val): # ^^^^^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__(). +# documentation ```python +# > (method) __init__: def __init__( +# > self: Self@Example, +# > in_val: Unknown +# > ) -> None: ... +# > ``` # ^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__().(self) +# documentation ```python +# > (parameter) self: Self@Example +# > ``` # ^^^^^^ definition snapshot-util 0.1 class_nohint/Example#__init__().(in_val) +# documentation ```python +# > (parameter) in_val: Unknown +# > ``` self.x = in_val # ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) # ^ definition snapshot-util 0.1 class_nohint/Example#x. +# documentation ```python +# > (attribute) x: Unknown +# > ``` # ^^^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(in_val) self.x = self.x + 1 # ^^^^ reference snapshot-util 0.1 class_nohint/Example#__init__().(self) @@ -29,9 +80,36 @@ def __init__(self, in_val): def something(self): # ^^^^^^^^^ definition snapshot-util 0.1 class_nohint/Example#something(). +# documentation ```python +# > (method) something: def something(self: ... +# > ``` # ^^^^ definition snapshot-util 0.1 class_nohint/Example#something().(self) +# documentation ```python +# > (parameter) self: Self@Example +# > ``` print(self.x) # ^^^^^ reference python-stdlib 3.13 builtins/print(). +# external documentation ```python +# > (function) print: def print( +# > *values: object, +# > *, +# > sep : str | None =... +# > end : str | None =... +# > file : SupportsWrite[str] | None =... +# > flush : Literal[False] =... +# > ) -> None: ... +# > ``` +# > --- +# > Prints the values to a stream, or to sys... +# > +# > sep +# >   string inserted between valu... +# > end +# >   string appended after the la... +# > file +# >   a file-like object (stream);... +# > flush +# >   whether to forcibly flush th... # ^^^^ reference snapshot-util 0.1 class_nohint/Example#something().(self) # ^ reference snapshot-util 0.1 class_nohint/Example#x. print(self.y) diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py index 66fdcc5e18..8bb7a347f0 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/field_docstring.py @@ -1,12 +1,50 @@ # < definition scip-python python snapshot-util 0.1 field_docstring/__init__: +#documentation ```python +# > (module) field_docstring +# > ``` # format-options: showDocs class ClassWithField: # ^^^^^^^^^^^^^^ definition snapshot-util 0.1 field_docstring/ClassWithField# +# documentation ```python +# > (class) ClassWithField: type[ClassWithFi... +# > ``` a: int # ^ definition snapshot-util 0.1 field_docstring/ClassWithField#a. +# documentation ```python +# > (variable) a: int +# > ``` +# > --- +# > Hello world, this is a # ^^^ reference python-stdlib 3.13 builtins/int# +# external documentation ```python +# > (class) int: +# > @overload +# > def __new__(x: ConvertibleToInt = 0, /) ... +# > def __new__( +# > x: bytearray | bytes | str, +# > /, +# > base: SupportsIndex +# > ) -> int: ... +# > ``` +# > --- +# > int([x]) -> integer +# > int(x, base=10) -> integer +# > +# > Convert a number or string to an integer... +# > are given. If x is a number, return x._... +# > numbers, this truncates towards zero. +# > +# > If x is not a number or if base is given... +# > bytes, or bytearray instance representin... +# > given base. The literal can be preceded... +# > by whitespace. The base defaults to 10.... +# > Base 0 means to interpret the base from ... +# > ```python +# > >>> int('0b100', base=0) +# > ``` +# > 4 """Hello world, this is a""" diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py index af26724502..6992aeefec 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inferred_field_docstring.py @@ -1,16 +1,64 @@ # < definition scip-python python snapshot-util 0.1 inferred_field_docstring/__init__: +#documentation ```python +# > (module) inferred_field_docstring +# > ``` # format-options: showDocs class ClassWithInferredField: # ^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField# +# documentation ```python +# > (class) ClassWithInferredField: type[Cla... +# > ``` def __init__(self, b: int): # ^^^^^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__(). +# documentation ```python +# > (method) __init__: def __init__( +# > self: Self@ClassWithInferredField, +# > b: int +# > ) -> None: ... +# > ``` # ^^^^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(self) +# documentation ```python +# > (parameter) self: Self@ClassWithInferred... +# > ``` # ^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(b) +# documentation ```python +# > (parameter) b: int +# > ``` # ^^^ reference python-stdlib 3.13 builtins/int# +# external documentation ```python +# > (class) int: +# > @overload +# > def __new__(x: ConvertibleToInt = 0, /) ... +# > def __new__( +# > x: bytearray | bytes | str, +# > /, +# > base: SupportsIndex +# > ) -> int: ... +# > ``` +# > --- +# > int([x]) -> integer +# > int(x, base=10) -> integer +# > +# > Convert a number or string to an integer... +# > are given. If x is a number, return x._... +# > numbers, this truncates towards zero. +# > +# > If x is not a number or if base is given... +# > bytes, or bytearray instance representin... +# > given base. The literal can be preceded... +# > by whitespace. The base defaults to 10.... +# > Base 0 means to interpret the base from ... +# > ```python +# > >>> int('0b100', base=0) +# > ``` +# > 4 self.b = b # ^^^^ reference snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(self) # ^ definition snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#b. +# documentation ```python +# > (attribute) b: int +# > ``` # ^ reference snapshot-util 0.1 inferred_field_docstring/ClassWithInferredField#__init__().(b) diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py b/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py index 68ac39fa05..16cda68f8a 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/module_docstring.py @@ -1,4 +1,9 @@ # < definition scip-python python snapshot-util 0.1 module_docstring/__init__: +#documentation ```python +# > (module) module_docstring +# > ``` +# > --- +# > This is a docstring for this module # format-options: showDocs @@ -8,5 +13,8 @@ def x(): # ^ definition snapshot-util 0.1 module_docstring/x(). +# documentation ```python +# > (function) x: def x() -> Literal[1]: ... +# > ``` return 1 diff --git a/pyrefly/lib/test/lsp/hover.rs b/pyrefly/lib/test/lsp/hover.rs index 0f56446078..e3a79b50bc 100644 --- a/pyrefly/lib/test/lsp/hover.rs +++ b/pyrefly/lib/test/lsp/hover.rs @@ -18,6 +18,7 @@ use pyrefly_python::module_path::ModulePath; use ruff_text_size::TextSize; use crate::lsp::wasm::hover::HoverOptions; +use crate::lsp::wasm::hover::HoverResult; use crate::lsp::wasm::hover::get_hover; use crate::lsp::wasm::hover::get_hover_with_verbosity; use crate::state::require::Require; @@ -50,6 +51,7 @@ fn get_test_report_at_verbosity( HoverOptions { show_go_to_links: true, verbosity_level, + field_docstring_at_definition: false, }, ) { Some(result) => match result.hover { @@ -1199,6 +1201,43 @@ widget.box ); } +#[test] +fn hover_shows_field_docstring_at_definition_when_enabled() { + let code = r#" +class Widget: + name: str +# ^ + """Name of the widget.""" +"#; + let mut env = TestEnv::new(); + env.add("main", code); + let (state, handle_for_name) = env.to_state(); + let handle = handle_for_name("main"); + let position = extract_cursors_for_test(code)[0]; + let hover = |field_docstring_at_definition| { + let options = HoverOptions { + show_go_to_links: false, + verbosity_level: 0, + field_docstring_at_definition, + }; + match get_hover_with_verbosity(&state.transaction(), &handle, position, options) { + Some(HoverResult { + hover: + Hover { + contents: Contents::MarkupContent(markup), + .. + }, + .. + }) => markup.value, + _ => "None".to_owned(), + } + }; + let enabled = hover(true); + assert!(enabled.contains("Name of the widget."), "got: {enabled}"); + let disabled = hover(false); + assert!(!disabled.contains("Name of the widget."), "got: {disabled}"); +} + #[test] fn hover_parameter_doc_with_complex_types() { let code = r#" From 9f18673bc53aee84ad0879863108f65ff59e11d9 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Thu, 22 Oct 2026 09:53:48 -0400 Subject: [PATCH 11/21] perf --- pyrefly/lib/commands/check.rs | 31 +++---- pyrefly/lib/report/scip.rs | 153 +++++++++++++++++++++++----------- pyrefly/lib/state/state.rs | 9 ++ 3 files changed, 130 insertions(+), 63 deletions(-) diff --git a/pyrefly/lib/commands/check.rs b/pyrefly/lib/commands/check.rs index a6bc62773b..f5ef8d90ad 100644 --- a/pyrefly/lib/commands/check.rs +++ b/pyrefly/lib/commands/check.rs @@ -35,7 +35,6 @@ use dupe::Dupe as _; use percent_encoding::AsciiSet; use percent_encoding::CONTROLS; use percent_encoding::utf8_percent_encode; -use protobuf::Message as _; use pyrefly_build::handle::Handle; use pyrefly_config::args::ConfigOverrideArgs; use pyrefly_config::base::Preset; @@ -2280,21 +2279,22 @@ impl CheckArgs { } } if let Some(path) = &self.output.report_scip { - fs_anyhow::write( - path, - report::scip::scip( - transaction, - handles, - &relative_to, - &report::scip::Package::project( - &relative_to, - self.output.report_scip_project_name.clone(), - self.output.report_scip_project_version.clone(), - )?, - version, - ) - .write_to_bytes()?, + let project = report::scip::Package::project( + &relative_to, + self.output.report_scip_project_name.clone(), + self.output.report_scip_project_version.clone(), )?; + let mut writer = BufWriter::new(File::create(path)?); + report::scip::write_scip( + &mut writer, + transaction, + handles, + &relative_to, + &project, + version, + ) + .and_then(|()| Ok(writer.flush()?)) + .with_context(|| format!("while writing SCIP index to `{}`", path.display()))?; } if let Some(pysa_reporter) = transaction.take_pysa_reporter() { report::pysa::write_project_file(&pysa_reporter, transaction, handles, &output_errors)?; @@ -2346,6 +2346,7 @@ mod tests { use std::path::PathBuf; use std::sync::Arc; + use protobuf::Message as _; use pyrefly_config::config::ConfigScope; use pyrefly_python::module::Module; use pyrefly_python::module_name::ModuleName; diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 675bc76f71..03c3a184fe 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -6,7 +6,9 @@ */ use std::collections::HashMap; +use std::io::Write; use std::iter; +use std::mem; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -15,6 +17,7 @@ use dupe::Dupe; use lsp_types::Contents; use lsp_types::Hover; use lsp_types::Uri; +use protobuf::Message as _; use pyrefly_build::handle::Handle; use pyrefly_python::ast::Ast; use pyrefly_python::docstring::Docstring; @@ -28,8 +31,10 @@ use pyrefly_util::absolutize::Absolutize; use pyrefly_util::fs_anyhow; use pyrefly_util::lined_buffer::DisplayPos; use pyrefly_util::lined_buffer::DisplayRange; +use pyrefly_util::lock::Mutex; use pyrefly_util::unix_path::path_to_unix_string; use pyrefly_util::visit::Visit; +use rayon::prelude::*; use ruff_python_ast::Expr; use ruff_python_ast::ExprContext; use ruff_python_ast::Identifier; @@ -109,38 +114,26 @@ impl Package { } } -pub fn scip( +/// Writes a SCIP index as a stream of `Index` messages, which a reader merges into one: the +/// metadata, then each document as soon as it is built, in no particular order, then the +/// external symbols. +pub fn write_scip( + writer: &mut (impl Write + Send), transaction: &Transaction, handles: &[Handle], project_root: &Path, project: &Package, version: &str, -) -> Index { +) -> anyhow::Result<()> { let project_root = project_root.absolutize(); - let mut indexer = Indexer { + let indexer = Indexer { transaction, project_root: &project_root, project, - definitions: HashMap::new(), - distributions: HashMap::new(), - external_symbols: HashMap::new(), + definitions: Mutex::new(HashMap::new()), + distributions: Mutex::new(HashMap::new()), + external_symbols: Mutex::new(HashMap::new()), }; - let mut documents: Vec = handles - .iter() - .filter_map(|handle| indexer.document(handle)) - .collect(); - documents.sort_by(|a, b| a.relative_path.cmp(&b.relative_path)); - let mut external_symbols: Vec = indexer - .external_symbols - .into_iter() - .filter(|(_, documentation)| !documentation.is_empty()) - .map(|(symbol, documentation)| SymbolInformation { - symbol, - documentation, - ..Default::default() - }) - .collect(); - external_symbols.sort_by(|a, b| a.symbol.cmp(&b.symbol)); Index { metadata: Some(Metadata { project_root: Uri::from_directory_path(&project_root) @@ -156,10 +149,47 @@ pub fn scip( ..Default::default() }) .into(), - documents, + ..Default::default() + } + .write_to_writer(writer)?; + let documents_writer = Mutex::new(&mut *writer); + transaction.install(|| { + handles.par_iter().try_for_each(|handle| { + let Some(document) = indexer.document(handle) else { + return anyhow::Ok(()); + }; + let bytes = Index { + documents: vec![document], + ..Default::default() + } + .write_to_bytes()?; + documents_writer.lock().write_all(&bytes)?; + Ok(()) + }) + })?; + let first_uses: Vec<_> = mem::take(&mut *indexer.external_symbols.lock()) + .into_iter() + .collect(); + let mut external_symbols: Vec = transaction.install(|| { + first_uses + .into_par_iter() + .filter_map(|(symbol, first_use)| { + let documentation = indexer.documentation(&first_use.handle, first_use.position); + (!documentation.is_empty()).then_some(SymbolInformation { + symbol, + documentation, + ..Default::default() + }) + }) + .collect() + }); + external_symbols.sort_by(|a, b| a.symbol.cmp(&b.symbol)); + Index { external_symbols, ..Default::default() } + .write_to_writer(writer)?; + Ok(()) } struct Indexer<'a> { @@ -167,15 +197,23 @@ struct Indexer<'a> { project_root: &'a Path, project: &'a Package, /// The definitions of every module seen so far, keyed by the range of the defining identifier. - definitions: HashMap<(ModuleName, ModulePath), Arc>>, + definitions: Mutex>>>, /// The installed distributions of every site-packages directory seen so far. - distributions: HashMap>, - /// The documentation of every symbol defined outside the project and referenced so far. - external_symbols: HashMap>, + distributions: Mutex>>>, + /// The first use, in path and position order, of every symbol defined outside the project and + /// referenced so far. The symbol is documented with the hover there, so its documentation + /// doesn't depend on the order documents are built in. + external_symbols: Mutex>, +} + +struct ExternalUse { + relative_path: PathBuf, + position: TextSize, + handle: Handle, } impl Indexer<'_> { - fn document(&mut self, handle: &Handle) -> Option { + fn document(&self, handle: &Handle) -> Option { let module = self.transaction.get_module_info(handle)?; let relative_path = module .path() @@ -226,9 +264,26 @@ impl Indexer<'_> { for x in uses { let symbol = match (self.reference(handle, &module, &x), x.kind) { (Some((symbol, external)), _) => { - if external && !self.external_symbols.contains_key(&symbol) { - let documentation = self.documentation(handle, use_position(&x)); - self.external_symbols.insert(symbol.clone(), documentation); + if external { + let position = use_position(&x); + let mut external_symbols = self.external_symbols.lock(); + let first_use = + external_symbols + .entry(symbol.clone()) + .or_insert_with(|| ExternalUse { + relative_path: relative_path.to_owned(), + position, + handle: handle.dupe(), + }); + if (relative_path, position) + < (first_use.relative_path.as_path(), first_use.position) + { + *first_use = ExternalUse { + relative_path: relative_path.to_owned(), + position, + handle: handle.dupe(), + }; + } } symbol } @@ -315,7 +370,7 @@ impl Indexer<'_> { /// the same name found through the bases, in order. A method whose name starts with `__` /// implements nothing. fn implementations( - &mut self, + &self, handle: &Handle, module: &Module, class: &StmtClassDef, @@ -393,7 +448,7 @@ impl Indexer<'_> { /// The symbol a use refers to, and whether it is defined outside the project, or `None` if /// it has no definition we can name. - fn reference(&mut self, handle: &Handle, module: &Module, x: &Use) -> Option<(String, bool)> { + fn reference(&self, handle: &Handle, module: &Module, x: &Use) -> Option<(String, bool)> { let preference = preference(); let definition = match x.kind { UseKind::Name => self @@ -414,7 +469,7 @@ impl Indexer<'_> { /// The symbol of a definition, and whether it is defined outside the project, or `None` if /// it is a local of another module or has no definition we can name. fn symbol( - &mut self, + &self, handle: &Handle, module: &Module, definition: FindDefinitionItemWithDocstring, @@ -443,23 +498,23 @@ impl Indexer<'_> { } fn definitions( - &mut self, + &self, module: &Module, sys_info: SysInfo, ) -> Option>> { let key = (module.name(), module.path().dupe()); - if let Some(definitions) = self.definitions.get(&key) { + if let Some(definitions) = self.definitions.lock().get(&key) { return Some(definitions.dupe()); } let handle = Handle::new(module.name(), module.path().dupe(), sys_info); let ast = self.transaction.get_ast(&handle)?; let table: Arc> = Arc::new(definitions(module.name(), &ast).into_iter().collect()); - self.definitions.insert(key, table.dupe()); + self.definitions.lock().insert(key, table.dupe()); Some(table) } - fn package(&mut self, module: &Module, sys_info: SysInfo) -> Package { + fn package(&self, module: &Module, sys_info: SysInfo) -> Package { match module.path().details() { ModulePathDetails::BundledTypeshed(_) => { let version = sys_info.version(); @@ -478,11 +533,13 @@ impl Indexer<'_> { .iter() .flat_map(|config| config.site_package_path()) .find_map(|root| { - self.distributions + let installed = self + .distributions + .lock() .entry(root.clone()) - .or_insert_with(|| distributions(root)) - .get(&top_level) - .cloned() + .or_insert_with(|| Arc::new(distributions(root))) + .dupe(); + installed.get(&top_level).cloned() }) // Without package metadata, name the package after the top-level module. .unwrap_or_else(|| Package { @@ -701,8 +758,6 @@ fn module_symbol(package: &Package, module: ModuleName) -> String { #[cfg(test)] mod tests { - use protobuf::Message as _; - use super::*; use crate::report::scip::snapshot::format_snapshot; use crate::state::require::Require; @@ -754,13 +809,17 @@ mod tests { .with_site_package_paths(vec![snapshots.join("site-packages")]) .with_default_require_level(Require::Everything) .to_state(); - scip( + let mut bytes = Vec::new(); + write_scip( + &mut bytes, &state.transaction(), &handles, &input_dir, &project_package(&snapshots, case), "test", ) + .unwrap(); + Index::parse_from_bytes(&bytes).unwrap() } fn assert_snapshot(case: &str) { @@ -796,9 +855,7 @@ mod tests { #[test] fn external_symbols_are_deduplicated() { // `print` and `int` are each referenced more than once. - let bytes = index("class_nohint").write_to_bytes().unwrap(); - let symbols: Vec = Index::parse_from_bytes(&bytes) - .unwrap() + let symbols: Vec = index("class_nohint") .external_symbols .into_iter() .map(|x| x.symbol) diff --git a/pyrefly/lib/state/state.rs b/pyrefly/lib/state/state.rs index d248481f9f..8c2dea2476 100644 --- a/pyrefly/lib/state/state.rs +++ b/pyrefly/lib/state/state.rs @@ -958,6 +958,15 @@ impl<'a> Transaction<'a> { self.with_module_inner(handle, |x| x.get_ast()) } + /// Run `op` on the state's thread pool, which has an increased stack size. + pub fn install(&self, op: OP) -> R + where + OP: FnOnce() -> R + Send, + R: Send, + { + self.data.state.threads.install(op) + } + pub(crate) fn get_parsed_module(&self, handle: &Handle) -> Option> { self.with_module_inner(handle, |x| x.get_parsed_module()) } From 694333167a922426bb54fb7f89134c6c397cd6bd Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sun, 25 Oct 2026 22:41:31 -0400 Subject: [PATCH 12/21] more perf --- crates/pyrefly_python/src/ast.rs | 27 ++++++++++++++++++++++++++ pyrefly/lib/lsp/wasm/hover.rs | 3 +-- pyrefly/lib/lsp/wasm/signature_help.rs | 5 ++++- pyrefly/lib/state/lsp.rs | 2 +- 4 files changed, 33 insertions(+), 4 deletions(-) diff --git a/crates/pyrefly_python/src/ast.rs b/crates/pyrefly_python/src/ast.rs index 9f17943989..c92df592b6 100644 --- a/crates/pyrefly_python/src/ast.rs +++ b/crates/pyrefly_python/src/ast.rs @@ -45,6 +45,7 @@ use ruff_python_ast::visitor; use ruff_python_ast::visitor::Visitor; use ruff_python_ast::visitor::source_order::SourceOrderVisitor; use ruff_python_ast::visitor::source_order::TraversalSignal; +use ruff_python_ast::visitor::source_order::walk_stmt; use ruff_python_parser::ParseError; use ruff_python_parser::ParseOptions; use ruff_python_parser::Parsed; @@ -469,6 +470,32 @@ impl Ast { ) } + /// Like `module.visit`, but only calls `f` on expressions that contain `position`, and + /// skips statements that don't contain it. + pub fn visit_covering_exprs<'a>( + module: &'a ModModule, + position: TextSize, + f: &mut dyn FnMut(&'a Expr), + ) { + struct X<'a, 'f> { + position: TextSize, + f: &'f mut dyn FnMut(&'a Expr), + } + impl<'a> SourceOrderVisitor<'a> for X<'a, '_> { + fn visit_stmt(&mut self, x: &'a Stmt) { + if x.range().contains_inclusive(self.position) { + walk_stmt(self, x); + } + } + fn visit_expr(&mut self, x: &'a Expr) { + if x.range().contains_inclusive(self.position) { + (self.f)(x); + } + } + } + X { position, f }.visit_body(&module.body); + } + /// Given a module and a position, find all AST nodes that "cover" the position. /// Return a vector of AST nodes sorted by the node's range, where the "innermost" node that covers the /// position comes first, and parent nodes of that covering node come later. diff --git a/pyrefly/lib/lsp/wasm/hover.rs b/pyrefly/lib/lsp/wasm/hover.rs index cb88eb44a2..06466ff470 100644 --- a/pyrefly/lib/lsp/wasm/hover.rs +++ b/pyrefly/lib/lsp/wasm/hover.rs @@ -38,7 +38,6 @@ use pyrefly_types::type_var::Variance; use pyrefly_types::types::Type; use pyrefly_util::absolutize::Absolutize as _; use pyrefly_util::lined_buffer::LineNumber; -use pyrefly_util::visit::Visit; use regex::Regex; use ruff_python_ast::AnyNodeRef; use ruff_python_ast::Identifier; @@ -840,7 +839,7 @@ fn resolve_hovered_type( use ruff_python_ast::Expr; let ast = ast?; let mut result = None; - ast.visit(&mut |expr: &Expr| { + Ast::visit_covering_exprs(ast, position, &mut |expr: &Expr| { if let Expr::Call(call) = expr && call.func.range().contains(position) { diff --git a/pyrefly/lib/lsp/wasm/signature_help.rs b/pyrefly/lib/lsp/wasm/signature_help.rs index 7ac9f22a3a..1a57becae8 100644 --- a/pyrefly/lib/lsp/wasm/signature_help.rs +++ b/pyrefly/lib/lsp/wasm/signature_help.rs @@ -17,6 +17,7 @@ use lsp_types::ParameterInformationLabel; use lsp_types::SignatureHelp; use lsp_types::SignatureInformation; use pyrefly_build::handle::Handle; +use pyrefly_python::ast::Ast; use pyrefly_python::docstring::Docstring; use pyrefly_python::docstring::parse_parameter_documentation; use pyrefly_python::module::Module; @@ -224,7 +225,9 @@ impl Transaction<'_> { ) -> Option { let mod_module = self.get_ast(handle)?; let mut res = None; - mod_module.visit(&mut |x| Self::visit_finding_signature_range(x, position, &mut res)); + Ast::visit_covering_exprs(&mod_module, position, &mut |x| { + Self::visit_finding_signature_range(x, position, &mut res) + }); let (callee_range, call_args_range, mut active_argument, provided_arg_ranges) = res?; // When the cursor is in the argument list but not inside any argument yet, // estimate the would-be positional index by counting commas up to the cursor. diff --git a/pyrefly/lib/state/lsp.rs b/pyrefly/lib/state/lsp.rs index 36c7e10b99..a1a27905dd 100644 --- a/pyrefly/lib/state/lsp.rs +++ b/pyrefly/lib/state/lsp.rs @@ -1243,7 +1243,7 @@ impl<'a> Transaction<'a> { } } let mut res = None; - mod_module.visit(&mut |x| f(x, position, &mut res)); + Ast::visit_covering_exprs(&mod_module, position, &mut |x| f(x, position, &mut res)); res } From 91aecca5dc41918b0c53ba0df8c81f36fc7fc5a1 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Tue, 27 Oct 2026 23:21:01 -0400 Subject: [PATCH 13/21] special forms and installed distribution priority --- pyrefly/lib/report/scip.rs | 89 ++++++++++++------- .../snapshots/input/unique/stdlib_backport.py | 5 ++ .../output/unique/builtin_import_refs.py | 4 +- .../snapshots/output/unique/inherits_class.py | 2 +- .../output/unique/stdlib_backport.py | 12 +++ .../METADATA | 3 + .../typing_extensions-4.16.0.dist-info/RECORD | 3 + 7 files changed, 83 insertions(+), 35 deletions(-) create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/stdlib_backport.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/stdlib_backport.py create mode 100644 pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/METADATA create mode 100644 pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/RECORD diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 03c3a184fe..75d5a1291a 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -192,6 +192,14 @@ pub fn write_scip( Ok(()) } +/// Top-level names whose kind in the bundled stubs differs from how other indexers name them: +/// `Any` was a variable in older stubs, and `NotImplemented` and `Ellipsis` are named as types. +const SPECIAL_FORMS: &[(&str, &str, Suffix)] = &[ + ("typing", "Any", Suffix::Term), + ("builtins", "NotImplemented", Suffix::Type), + ("builtins", "Ellipsis", Suffix::Type), +]; + struct Indexer<'a> { transaction: &'a Transaction<'a>, project_root: &'a Path, @@ -479,7 +487,7 @@ impl Indexer<'_> { .path() .as_path() .starts_with(self.project_root); - let package = self.package(&definition.module, handle.sys_info().dupe()); + let package = self.package(&definition.module, handle); // A submodule accessed as an attribute resolves to the empty range at the module's start. if definition.metadata.symbol_kind() == Some(SymbolKind::Module) || definition.definition_range.is_empty() @@ -508,46 +516,63 @@ impl Indexer<'_> { } let handle = Handle::new(module.name(), module.path().dupe(), sys_info); let ast = self.transaction.get_ast(&handle)?; - let table: Arc> = - Arc::new(definitions(module.name(), &ast).into_iter().collect()); + let mut table: SmallMap<_, _> = definitions(module.name(), &ast).into_iter().collect(); + for (module_name, name, suffix) in SPECIAL_FORMS { + if module.name().as_str() != *module_name { + continue; + } + for symbol in table.values_mut() { + if let ScipSymbol::Global(descriptors) = symbol + && let [_, descriptor] = descriptors.as_mut_slice() + && descriptor.name == *name + { + descriptor.suffix = (*suffix).into(); + } + } + } + let table = Arc::new(table); self.definitions.lock().insert(key, table.dupe()); Some(table) } - fn package(&self, module: &Module, sys_info: SysInfo) -> Package { - match module.path().details() { - ModulePathDetails::BundledTypeshed(_) => { - let version = sys_info.version(); + /// The package of a module referenced from `handle`, whose site-packages are searched for + /// installed distributions. + fn package(&self, module: &Module, handle: &Handle) -> Package { + let bundled = matches!(module.path().details(), ModulePathDetails::BundledTypeshed(_)); + if !bundled && module.path().as_path().starts_with(self.project_root) { + return self.project.clone(); + } + // Match by top-level module, so that bundled stubs get the installed package too, even for + // a stdlib module like `typing_extensions`. + let top_level = module.name().first_component(); + let config = self.transaction.get_config(handle); + let installed = config + .iter() + .flat_map(|config| config.site_package_path()) + .find_map(|root| { + let installed = self + .distributions + .lock() + .entry(root.clone()) + .or_insert_with(|| Arc::new(distributions(root))) + .dupe(); + installed.get(&top_level).cloned() + }); + installed.unwrap_or_else(|| { + if bundled { + let version = handle.sys_info().version(); Package { name: "python-stdlib".to_owned(), version: format!("{}.{}", version.major, version.minor), } + } else { + // Without package metadata, name the package after the top-level module. + Package { + name: top_level.to_string(), + version: String::new(), + } } - _ if module.path().as_path().starts_with(self.project_root) => self.project.clone(), - _ => { - // Match by top-level module, so that bundled stubs get the installed package too. - let top_level = module.name().first_component(); - let handle = Handle::new(module.name(), module.path().dupe(), sys_info); - let config = self.transaction.get_config(&handle); - config - .iter() - .flat_map(|config| config.site_package_path()) - .find_map(|root| { - let installed = self - .distributions - .lock() - .entry(root.clone()) - .or_insert_with(|| Arc::new(distributions(root))) - .dupe(); - installed.get(&top_level).cloned() - }) - // Without package metadata, name the package after the top-level module. - .unwrap_or_else(|| Package { - name: top_level.to_string(), - version: String::new(), - }) - } - } + }) } } diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/stdlib_backport.py b/pyrefly/lib/report/scip/snapshots/input/unique/stdlib_backport.py new file mode 100644 index 0000000000..17f5ee8de1 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/stdlib_backport.py @@ -0,0 +1,5 @@ +# typing_extensions is in the bundled stdlib stubs, but an installed distribution takes +# precedence. +from typing_extensions import Doc + +print(Doc) diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py index bb12da2e69..10f9e36703 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/builtin_import_refs.py @@ -2,9 +2,9 @@ from typing import Any # ^^^^^^ reference python-stdlib 3.13 typing/__init__: -# ^^^ reference python-stdlib 3.13 typing/Any# +# ^^^ reference python-stdlib 3.13 typing/Any. print(Any) #^^^^ reference python-stdlib 3.13 builtins/print(). -# ^^^ reference python-stdlib 3.13 typing/Any# +# ^^^ reference python-stdlib 3.13 typing/Any. diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py index 7edf578305..e85198f5f1 100644 --- a/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py +++ b/pyrefly/lib/report/scip/snapshots/output/unique/inherits_class.py @@ -7,7 +7,7 @@ def x(self) -> int: # ^^^^ definition snapshot-util 0.1 inherits_class/A#x().(self) # ^^^ reference python-stdlib 3.13 builtins/int# raise NotImplemented -# ^^^^^^^^^^^^^^ reference python-stdlib 3.13 builtins/NotImplemented. +# ^^^^^^^^^^^^^^ reference python-stdlib 3.13 builtins/NotImplemented# def matched_despite_different_type(self, x: int): # ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ definition snapshot-util 0.1 inherits_class/A#matched_despite_different_type(). diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/stdlib_backport.py b/pyrefly/lib/report/scip/snapshots/output/unique/stdlib_backport.py new file mode 100644 index 0000000000..360400da29 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/stdlib_backport.py @@ -0,0 +1,12 @@ +# < definition scip-python python snapshot-util 0.1 stdlib_backport/__init__: + +# typing_extensions is in the bundled stdlib stubs, but an installed distribution takes +# precedence. +from typing_extensions import Doc +# ^^^^^^^^^^^^^^^^^ reference typing_extensions 4.16.0 typing_extensions/__init__: +# ^^^ reference typing_extensions 4.16.0 typing_extensions/Doc# + +print(Doc) +#^^^^ reference python-stdlib 3.13 builtins/print(). +# ^^^ reference typing_extensions 4.16.0 typing_extensions/Doc# + diff --git a/pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/METADATA b/pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/METADATA new file mode 100644 index 0000000000..134c671c8e --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Metadata-Version: 2.1 +Name: typing_extensions +Version: 4.16.0 diff --git a/pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/RECORD b/pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/RECORD new file mode 100644 index 0000000000..f181eafd19 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/site-packages/typing_extensions-4.16.0.dist-info/RECORD @@ -0,0 +1,3 @@ +typing_extensions.py,, +typing_extensions-4.16.0.dist-info/METADATA,, +typing_extensions-4.16.0.dist-info/RECORD,, From 7d7aa3ad6af0a7b618c463e2d714b993f871e018 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Wed, 28 Oct 2026 17:02:38 -0400 Subject: [PATCH 14/21] more parity fixes --- pyrefly/lib/report/scip.rs | 26 +++++- .../scip/snapshots/input/unique/rebinding.py | 41 +++++++++ .../snapshots/output/comprehensions/comp.py | 6 +- .../scip/snapshots/output/unique/rebinding.py | 85 +++++++++++++++++++ 4 files changed, 153 insertions(+), 5 deletions(-) create mode 100644 pyrefly/lib/report/scip/snapshots/input/unique/rebinding.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/unique/rebinding.py diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 75d5a1291a..0b5d641833 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -59,6 +59,7 @@ use scip::types::TextEncoding; use scip::types::ToolInfo; use scip::types::descriptor::Suffix; use starlark_map::small_map::SmallMap; +use starlark_map::small_set::SmallSet; use crate::config::config::ConfigFile; use crate::lsp::wasm::hover::HoverOptions; @@ -240,13 +241,27 @@ impl Indexer<'_> { ast.body .iter() .for_each(|x| collect_enclosing_ranges(x, &mut enclosing_ranges)); + // Only the first binding of a symbol defines it, and later ones refer to it. A parameter + // is defined again by each overload. + let mut parameters = SmallSet::new(); + ast.body + .iter() + .for_each(|x| collect_parameters(x, &mut parameters)); + let mut defined = SmallSet::new(); occurrences.extend(definitions.iter().map(|(range, symbol)| { + let symbol = format_scip_symbol(self.project, symbol); + let is_definition = defined.insert(symbol.clone()) || parameters.contains(range); Occurrence { range: scip_range(&module, *range), - symbol: format_scip_symbol(self.project, symbol), - symbol_roles: SymbolRole::Definition as i32, + symbol, + symbol_roles: if is_definition { + SymbolRole::Definition as i32 + } else { + 0 + }, enclosing_range: enclosing_ranges .get(range) + .filter(|_| is_definition) .map_or_else(Vec::new, |x| scip_range(&module, *x)), ..Default::default() } @@ -690,6 +705,13 @@ fn collect_classes<'a>(x: &'a Stmt, classes: &mut Vec<&'a StmtClassDef>) { x.recurse(&mut |x| collect_classes(x, classes)); } +fn collect_parameters(x: &Stmt, ranges: &mut SmallSet) { + if let Stmt::FunctionDef(x) = x { + ranges.extend(x.parameters.iter().map(|x| x.name().range)); + } + x.recurse(&mut |x| collect_parameters(x, ranges)); +} + /// Maps the name of every function and class to its whole definition, decorators included. fn collect_enclosing_ranges(x: &Stmt, ranges: &mut SmallMap) { match x { diff --git a/pyrefly/lib/report/scip/snapshots/input/unique/rebinding.py b/pyrefly/lib/report/scip/snapshots/input/unique/rebinding.py new file mode 100644 index 0000000000..a90daa4ce0 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/unique/rebinding.py @@ -0,0 +1,41 @@ +# Only the first binding of a symbol is its definition, except that every overload defines its +# parameters. +from typing import overload + +x = 1 +x = 2 + + +def f(): + y = 1 + y = 2 + return y + + +class C: + a = 1 + a = 2 + + def __init__(self): + self.b = 1 + self.b = 2 + + @overload + def m(self, v: int) -> int: ... + @overload + def m(self, v: str) -> str: ... + def m(self, v): + return v + + +def g(): ... +def g(): ... + + +def h(p): + p = 1 + return p + + +for x in range(3): + pass diff --git a/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py index 15b9f6db1b..a70c6ed30c 100644 --- a/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py +++ b/pyrefly/lib/report/scip/snapshots/output/comprehensions/comp.py @@ -11,18 +11,18 @@ def something(x): # ^ reference local 0 # ^ definition local 0 _ = [something(x) for x in "should be local 1"] -# definition snapshot-util 0.1 comp/_. +# reference snapshot-util 0.1 comp/_. # ^^^^^^^^^ reference snapshot-util 0.1 comp/something(). # ^ reference local 1 # ^ definition local 1 _ = [something(x) for x in "should be local 2" if x == "s"] -# definition snapshot-util 0.1 comp/_. +# reference snapshot-util 0.1 comp/_. # ^^^^^^^^^ reference snapshot-util 0.1 comp/something(). # ^ reference local 2 # ^ definition local 2 # ^ reference local 2 _ = {k: x for (k, x) in enumerate(range(10))} -# definition snapshot-util 0.1 comp/_. +# reference snapshot-util 0.1 comp/_. # ^ reference local 3 # ^ reference local 4 # ^ definition local 3 diff --git a/pyrefly/lib/report/scip/snapshots/output/unique/rebinding.py b/pyrefly/lib/report/scip/snapshots/output/unique/rebinding.py new file mode 100644 index 0000000000..1c5b9295c7 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/unique/rebinding.py @@ -0,0 +1,85 @@ +# < definition scip-python python snapshot-util 0.1 rebinding/__init__: + +# Only the first binding of a symbol is its definition, except that every overload defines its +# parameters. +from typing import overload +# ^^^^^^ reference python-stdlib 3.13 typing/__init__: +# ^^^^^^^^ reference python-stdlib 3.13 typing/overload(). + +x = 1 +# definition snapshot-util 0.1 rebinding/x. +x = 2 +# reference snapshot-util 0.1 rebinding/x. + + +def f(): +# ^ definition snapshot-util 0.1 rebinding/f(). + y = 1 +# ^ definition local 0 + y = 2 +# ^ reference local 0 + return y +# ^ reference local 0 + + +class C: +# ^ definition snapshot-util 0.1 rebinding/C# + a = 1 +# ^ definition snapshot-util 0.1 rebinding/C#a. + a = 2 +# ^ reference snapshot-util 0.1 rebinding/C#a. + + def __init__(self): +# ^^^^^^^^ definition snapshot-util 0.1 rebinding/C#__init__(). +# ^^^^ definition snapshot-util 0.1 rebinding/C#__init__().(self) + self.b = 1 +# ^^^^ reference snapshot-util 0.1 rebinding/C#__init__().(self) +# ^ definition snapshot-util 0.1 rebinding/C#b. + self.b = 2 +# ^^^^ reference snapshot-util 0.1 rebinding/C#__init__().(self) +# ^ reference snapshot-util 0.1 rebinding/C#b. + + @overload +# ^^^^^^^^ reference python-stdlib 3.13 typing/overload(). + def m(self, v: int) -> int: ... +# ^ definition snapshot-util 0.1 rebinding/C#m(). +# ^^^^ definition snapshot-util 0.1 rebinding/C#m().(self) +# ^ definition snapshot-util 0.1 rebinding/C#m().(v) +# ^^^ reference python-stdlib 3.13 builtins/int# +# ^^^ reference python-stdlib 3.13 builtins/int# + @overload +# ^^^^^^^^ reference python-stdlib 3.13 typing/overload(). + def m(self, v: str) -> str: ... +# ^ reference snapshot-util 0.1 rebinding/C#m(). +# ^^^^ definition snapshot-util 0.1 rebinding/C#m().(self) +# ^ definition snapshot-util 0.1 rebinding/C#m().(v) +# ^^^ reference python-stdlib 3.13 builtins/str# +# ^^^ reference python-stdlib 3.13 builtins/str# + def m(self, v): +# ^ reference snapshot-util 0.1 rebinding/C#m(). +# ^^^^ definition snapshot-util 0.1 rebinding/C#m().(self) +# ^ definition snapshot-util 0.1 rebinding/C#m().(v) + return v +# ^ reference snapshot-util 0.1 rebinding/C#m().(v) + + +def g(): ... +# ^ definition snapshot-util 0.1 rebinding/g(). +def g(): ... +# ^ reference snapshot-util 0.1 rebinding/g(). + + +def h(p): +# ^ definition snapshot-util 0.1 rebinding/h(). +# ^ definition snapshot-util 0.1 rebinding/h().(p) + p = 1 +# ^ reference snapshot-util 0.1 rebinding/h().(p) + return p +# ^ reference snapshot-util 0.1 rebinding/h().(p) + + +for x in range(3): +# ^ reference snapshot-util 0.1 rebinding/x. +# ^^^^^ reference python-stdlib 3.13 builtins/range# + pass + From dd6601de6098f1f13ffaf96633ffb1edb03f5740 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Wed, 28 Oct 2026 17:45:39 -0400 Subject: [PATCH 15/21] handle implicit globals --- pyrefly/lib/report/scip.rs | 38 ++++++++++++++----- .../snapshots/output/dunder_vars/__main__.py | 2 +- 2 files changed, 30 insertions(+), 10 deletions(-) diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 0b5d641833..3e9d1b8f26 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -71,6 +71,7 @@ use crate::report::scip::symbols::descriptor; use crate::state::lsp::FindDefinitionItemWithDocstring; use crate::state::lsp::FindPreference; use crate::state::state::Transaction; +use crate::types::globals::ImplicitGlobal; #[cfg(test)] mod snapshot; @@ -474,10 +475,25 @@ impl Indexer<'_> { fn reference(&self, handle: &Handle, module: &Module, x: &Use) -> Option<(String, bool)> { let preference = preference(); let definition = match x.kind { - UseKind::Name => self - .transaction - .find_definition_for_name_use(handle, &x.identifier, preference) - .ok()?, + UseKind::Name => { + let definition = self + .transaction + .find_definition_for_name_use(handle, &x.identifier, preference) + .ok() + .flatten(); + // Implicit module globals like `__name__` have no definition, so they are named + // as stdlib builtins. + if definition.is_none() + && ImplicitGlobal::implicit_globals(false).any(|g| *g.name() == x.identifier.id) + { + let descriptors = vec![ + descriptor("builtins", Suffix::Namespace), + descriptor(&x.identifier.id, Suffix::Type), + ]; + return Some((global_symbol(&stdlib_package(handle), descriptors), true)); + } + definition + } UseKind::Import | UseKind::Attribute | UseKind::Keyword => { let definitions = self .transaction @@ -575,11 +591,7 @@ impl Indexer<'_> { }); installed.unwrap_or_else(|| { if bundled { - let version = handle.sys_info().version(); - Package { - name: "python-stdlib".to_owned(), - version: format!("{}.{}", version.major, version.minor), - } + stdlib_package(handle) } else { // Without package metadata, name the package after the top-level module. Package { @@ -591,6 +603,14 @@ impl Indexer<'_> { } } +fn stdlib_package(handle: &Handle) -> Package { + let version = handle.sys_info().version(); + Package { + name: "python-stdlib".to_owned(), + version: format!("{}.{}", version.major, version.minor), + } +} + /// The distributions installed in a site-packages directory, keyed by each top-level module they /// installed. A distribution whose metadata can't be read is left out. fn distributions(site_packages: &Path) -> HashMap { diff --git a/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py b/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py index 809609201e..9d1b2104c6 100644 --- a/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py +++ b/pyrefly/lib/report/scip/snapshots/output/dunder_vars/__main__.py @@ -2,7 +2,7 @@ if __name__ == '__main__': -# ^^^^^^^^ reference local 0 +# ^^^^^^^^ reference python-stdlib 3.13 builtins/__name__# print("main") # ^^^^^ reference python-stdlib 3.13 builtins/print(). From 354696c78da975a4d1a8865a2adbfa4a1faae24b Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Sun, 1 Nov 2026 21:51:58 -0500 Subject: [PATCH 16/21] perf improvements --- pyrefly/lib/commands/check.rs | 6 ++--- pyrefly/lib/report/scip.rs | 49 ++++++++++++++++++++++++++--------- 2 files changed, 40 insertions(+), 15 deletions(-) diff --git a/pyrefly/lib/commands/check.rs b/pyrefly/lib/commands/check.rs index f5ef8d90ad..eee5324b31 100644 --- a/pyrefly/lib/commands/check.rs +++ b/pyrefly/lib/commands/check.rs @@ -1778,10 +1778,10 @@ impl CheckArgs { let retain = self.output.report_binding_memory.is_some() || self.output.debug_info.is_some() || self.output.report_trace.is_some() - || self.output.report_glean.is_some() - || self.output.report_scip.is_some(); + || self.output.report_glean.is_some(); RequireLevels { - specified: if retain { + // SCIP solves the dependencies it refers to once the indexed files are written. + specified: if retain || self.output.report_scip.is_some() { Require::Everything } else { Require::Errors diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 3e9d1b8f26..573f5fd202 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -23,7 +23,6 @@ use pyrefly_python::ast::Ast; use pyrefly_python::docstring::Docstring; use pyrefly_python::module::Module; use pyrefly_python::module_name::ModuleName; -use pyrefly_python::module_path::ModulePath; use pyrefly_python::module_path::ModulePathDetails; use pyrefly_python::symbol_kind::SymbolKind; use pyrefly_python::sys_info::SysInfo; @@ -70,6 +69,7 @@ use crate::report::scip::symbols::definitions; use crate::report::scip::symbols::descriptor; use crate::state::lsp::FindDefinitionItemWithDocstring; use crate::state::lsp::FindPreference; +use crate::state::require::Require; use crate::state::state::Transaction; use crate::types::globals::ImplicitGlobal; @@ -121,7 +121,7 @@ impl Package { /// external symbols. pub fn write_scip( writer: &mut (impl Write + Send), - transaction: &Transaction, + transaction: &mut Transaction, handles: &[Handle], project_root: &Path, project: &Package, @@ -129,7 +129,7 @@ pub fn write_scip( ) -> anyhow::Result<()> { let project_root = project_root.absolutize(); let indexer = Indexer { - transaction, + transaction: &*transaction, project_root: &project_root, project, definitions: Mutex::new(HashMap::new()), @@ -169,6 +169,29 @@ pub fn write_scip( Ok(()) }) })?; + let Indexer { + definitions, + distributions, + external_symbols, + .. + } = indexer; + // A keyword argument is documented with its parameter's type, which comes from the defining + // module's answers. Those are only kept for modules solved to `Require::Everything`. + let dependencies: Vec = definitions + .lock() + .keys() + .filter(|x| transaction.get_ast(x).is_none()) + .cloned() + .collect(); + transaction.run(&dependencies, Require::Everything, None); + let indexer = Indexer { + transaction: &*transaction, + project_root: &project_root, + project, + definitions, + distributions, + external_symbols, + }; let first_uses: Vec<_> = mem::take(&mut *indexer.external_symbols.lock()) .into_iter() .collect(); @@ -207,7 +230,7 @@ struct Indexer<'a> { project_root: &'a Path, project: &'a Package, /// The definitions of every module seen so far, keyed by the range of the defining identifier. - definitions: Mutex>>>, + definitions: Mutex>>>, /// The installed distributions of every site-packages directory seen so far. distributions: Mutex>>>, /// The first use, in path and position order, of every symbol defined outside the project and @@ -541,12 +564,14 @@ impl Indexer<'_> { module: &Module, sys_info: SysInfo, ) -> Option>> { - let key = (module.name(), module.path().dupe()); - if let Some(definitions) = self.definitions.lock().get(&key) { + let handle = Handle::new(module.name(), module.path().dupe(), sys_info); + if let Some(definitions) = self.definitions.lock().get(&handle) { return Some(definitions.dupe()); } - let handle = Handle::new(module.name(), module.path().dupe(), sys_info); - let ast = self.transaction.get_ast(&handle)?; + // Only the indexed files keep their AST, so a dependency is parsed again. + let ast = self.transaction.get_ast(&handle).unwrap_or_else(|| { + Arc::new(Ast::parse(module.contents(), module.source_type()).0) + }); let mut table: SmallMap<_, _> = definitions(module.name(), &ast).into_iter().collect(); for (module_name, name, suffix) in SPECIAL_FORMS { if module.name().as_str() != *module_name { @@ -562,7 +587,7 @@ impl Indexer<'_> { } } let table = Arc::new(table); - self.definitions.lock().insert(key, table.dupe()); + self.definitions.lock().insert(handle, table.dupe()); Some(table) } @@ -825,9 +850,10 @@ fn module_symbol(package: &Package, module: ModuleName) -> String { #[cfg(test)] mod tests { + use pyrefly_python::module_path::ModulePath; + use super::*; use crate::report::scip::snapshot::format_snapshot; - use crate::state::require::Require; use crate::test::util::TestEnv; fn snapshots_dir() -> PathBuf { @@ -874,12 +900,11 @@ mod tests { } let (state, _) = env .with_site_package_paths(vec![snapshots.join("site-packages")]) - .with_default_require_level(Require::Everything) .to_state(); let mut bytes = Vec::new(); write_scip( &mut bytes, - &state.transaction(), + &mut state.transaction(), &handles, &input_dir, &project_package(&snapshots, case), From 3b944fa0a4a4298ed5f52d4aa91e444dc71baa4b Mon Sep 17 00:00:00 2001 From: Christoph Hegemann Date: Mon, 2 Nov 2026 11:14:26 +0100 Subject: [PATCH 17/21] adds reproductions snapshots for a few minor bugs --- pyrefly/lib/report/scip.rs | 19 ++++++++++++-- .../snapshots/input/lambda_walrus/main.py | 4 +++ .../input/nonlocal_skips_scope/main.py | 14 ++++++++++ .../input/project_site_packages/main.py | 5 ++++ .../METADATA | 3 +++ .../review_dependency-2.0.0.dist-info/RECORD | 3 +++ .../review_dependency/__init__.py | 2 ++ .../snapshots/input/type_parameters/main.py | 4 +++ .../snapshots/output/lambda_walrus/main.py | 11 ++++++++ .../output/nonlocal_skips_scope/main.py | 26 +++++++++++++++++++ .../output/project_site_packages/main.py | 12 +++++++++ .../snapshots/output/type_parameters/main.py | 12 +++++++++ 12 files changed, 113 insertions(+), 2 deletions(-) create mode 100644 pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/METADATA create mode 100644 pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/RECORD create mode 100644 pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency/__init__.py create mode 100644 pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py create mode 100644 pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 75d5a1291a..ff07083757 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -813,10 +813,15 @@ mod tests { fn index(case: &str) -> Index { let snapshots = snapshots_dir(); let input_dir = snapshots.join("input").join(case); + // Case-local dependencies model a virtual environment inside the project root. + let site_packages = input_dir.join("site-packages"); let mut env = TestEnv::new(); let mut handles = Vec::new(); for path in glob::glob(&format!("{}/**/*.py", input_dir.display())).unwrap() { let path = path.unwrap(); + if path.starts_with(&site_packages) { + continue; + } // As with an import root, modules under `src/` are named without the prefix. let relative = path .strip_prefix(input_dir.join("src")) @@ -831,7 +836,7 @@ mod tests { )); } let (state, _) = env - .with_site_package_paths(vec![snapshots.join("site-packages")]) + .with_site_package_paths(vec![site_packages, snapshots.join("site-packages")]) .with_default_require_level(Require::Everything) .to_state(); let mut bytes = Vec::new(); @@ -844,7 +849,13 @@ mod tests { "test", ) .unwrap(); - Index::parse_from_bytes(&bytes).unwrap() + let index = Index::parse_from_bytes(&bytes).unwrap(); + assert_eq!( + index.documents.len(), + handles.len(), + "missing snapshot documents" + ); + index } fn assert_snapshot(case: &str) { @@ -917,14 +928,18 @@ mod tests { f_string, file_from_module_import, graph_1278, + lambda_walrus, nameparts_import, nested_items, + nonlocal_skips_scope, odd_pkg_name_1, + project_site_packages, pyproject_1, relative_import, request_goofiness, single_class, single_function, + type_parameters, unique, unresolved_import, ); diff --git a/pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py b/pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py new file mode 100644 index 0000000000..3e369c1d4b --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py @@ -0,0 +1,4 @@ +# BUG: These lambda-local bindings share a module-level symbol. +# Each `hidden` should have a distinct local symbol. +first = lambda: (hidden := 1) +second = lambda: (hidden := 2) diff --git a/pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py b/pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py new file mode 100644 index 0000000000..89651d6e47 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py @@ -0,0 +1,14 @@ +# BUG: The nonlocal assignment has a different symbol from the outer binding. +# All occurrences of `x` should share the same symbol, skipping `middle`. +def outer(): + x = 0 + + def middle(): + def inner(): + nonlocal x + x = 1 + return x + + return inner + + return x, middle diff --git a/pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py new file mode 100644 index 0000000000..88ca166ada --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py @@ -0,0 +1,5 @@ +# BUG: This dependency inside the project root gets the project's package identity. +# Its symbols should belong to review-dependency 2.0.0. +from review_dependency import Widget + +widget = Widget() diff --git a/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/METADATA b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/METADATA new file mode 100644 index 0000000000..c62baf0e1e --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/METADATA @@ -0,0 +1,3 @@ +Metadata-Version: 2.1 +Name: review-dependency +Version: 2.0.0 diff --git a/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/RECORD b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/RECORD new file mode 100644 index 0000000000..7a3edb34b2 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency-2.0.0.dist-info/RECORD @@ -0,0 +1,3 @@ +review_dependency/__init__.py,, +review_dependency-2.0.0.dist-info/METADATA,, +review_dependency-2.0.0.dist-info/RECORD,, diff --git a/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency/__init__.py b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency/__init__.py new file mode 100644 index 0000000000..915991d800 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/site-packages/review_dependency/__init__.py @@ -0,0 +1,2 @@ +class Widget: + pass diff --git a/pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py b/pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py new file mode 100644 index 0000000000..974791f214 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py @@ -0,0 +1,4 @@ +# BUG: The type parameter has references but no definition occurrence. +# The declaration of `T` and both annotations should share a defined symbol. +def identity[T](x: T) -> T: + return x diff --git a/pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py b/pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py new file mode 100644 index 0000000000..f0ca866e25 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py @@ -0,0 +1,11 @@ +# < definition scip-python python snapshot-util 0.1 main/__init__: + +# BUG: These lambda-local bindings share a module-level symbol. +# Each `hidden` should have a distinct local symbol. +first = lambda: (hidden := 1) +#^^^^ definition snapshot-util 0.1 main/first. +# ^^^^^^ definition snapshot-util 0.1 main/hidden. +second = lambda: (hidden := 2) +#^^^^^ definition snapshot-util 0.1 main/second. +# ^^^^^^ definition snapshot-util 0.1 main/hidden. + diff --git a/pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py b/pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py new file mode 100644 index 0000000000..596736f52c --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py @@ -0,0 +1,26 @@ +# < definition scip-python python snapshot-util 0.1 main/__init__: + +# BUG: The nonlocal assignment has a different symbol from the outer binding. +# All occurrences of `x` should share the same symbol, skipping `middle`. +def outer(): +# ^^^^^ definition snapshot-util 0.1 main/outer(). + x = 0 +# ^ definition local 0 + + def middle(): +# ^^^^^^ definition snapshot-util 0.1 main/outer().middle(). + def inner(): +# ^^^^^ definition snapshot-util 0.1 main/outer().middle().inner(). + nonlocal x + x = 1 +# ^ definition local 1 + return x +# ^ reference local 1 + + return inner +# ^^^^^ reference snapshot-util 0.1 main/outer().middle().inner(). + + return x, middle +# ^ reference local 0 +# ^^^^^^ reference snapshot-util 0.1 main/outer().middle(). + diff --git a/pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py b/pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py new file mode 100644 index 0000000000..ce9b1ef210 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py @@ -0,0 +1,12 @@ +# < definition scip-python python snapshot-util 0.1 main/__init__: + +# BUG: This dependency inside the project root gets the project's package identity. +# Its symbols should belong to review-dependency 2.0.0. +from review_dependency import Widget +# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 review_dependency/__init__: +# ^^^^^^ reference snapshot-util 0.1 review_dependency/Widget# + +widget = Widget() +#^^^^^ definition snapshot-util 0.1 main/widget. +# ^^^^^^ reference snapshot-util 0.1 review_dependency/Widget# + diff --git a/pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py b/pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py new file mode 100644 index 0000000000..cb43213062 --- /dev/null +++ b/pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py @@ -0,0 +1,12 @@ +# < definition scip-python python snapshot-util 0.1 main/__init__: + +# BUG: The type parameter has references but no definition occurrence. +# The declaration of `T` and both annotations should share a defined symbol. +def identity[T](x: T) -> T: +# ^^^^^^^^ definition snapshot-util 0.1 main/identity(). +# ^ definition snapshot-util 0.1 main/identity().(x) +# ^ reference local 0 +# ^ reference local 0 + return x +# ^ reference snapshot-util 0.1 main/identity().(x) + From 9a45db4d49decd0e5b2f4ca9496b0d64009156de Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Mon, 2 Nov 2026 22:47:50 -0500 Subject: [PATCH 18/21] emit data as files are solved, evict answers when no longer needed --- pyrefly/lib/commands/check.rs | 69 +++++-- pyrefly/lib/report/scip.rs | 343 ++++++++++++++++++++-------------- pyrefly/lib/state/state.rs | 13 +- 3 files changed, 269 insertions(+), 156 deletions(-) diff --git a/pyrefly/lib/commands/check.rs b/pyrefly/lib/commands/check.rs index eee5324b31..55e35365c9 100644 --- a/pyrefly/lib/commands/check.rs +++ b/pyrefly/lib/commands/check.rs @@ -1591,7 +1591,7 @@ impl IncrementalCheckCommand { ) }); let defaults = args.output.resolve(config.as_deref()); - let run = args.prepare_cli_run(timings, transaction, handles, &defaults)?; + let run = args.prepare_cli_run(timings, transaction, version, handles, &defaults)?; check.run(|transaction, handles, sourcedb_errors| { let result = args.finish_cli_run( run, @@ -1638,6 +1638,7 @@ struct PreparedCliRun { memory_trace: MemoryUsageTrace, demand_tree_subscriber: Option, type_check_start: Instant, + scip: Option>>>, } impl CheckArgs { @@ -1810,7 +1811,7 @@ impl CheckArgs { require: Require, upsell: UpsellDecision, ) -> anyhow::Result<(CommandExitStatus, Vec)> { - let run = self.prepare_cli_run(timings, transaction, handles, defaults)?; + let run = self.prepare_cli_run(timings, transaction, version, handles, defaults)?; transaction.run(handles, require, None); self.finish_cli_run( run, @@ -1827,6 +1828,7 @@ impl CheckArgs { &self, timings: Timings, transaction: &mut Transaction, + version: &str, handles: &[Handle], defaults: &OutputDefaults, ) -> anyhow::Result { @@ -1887,6 +1889,41 @@ impl CheckArgs { transaction.set_cinderx_reporter(Some(cinderx_reporter)); } + // Each indexed file is written as soon as it is solved, while its answers are still in + // memory, so they don't have to be kept for the whole run. + let scip = match &self.output.report_scip { + Some(path) => { + let relative_to = self.output.relative_to.as_ref().map_or_else( + || std::env::current_dir().ok().unwrap_or_default(), + |x| PathBuf::from_str(x.as_str()).unwrap(), + ); + let project = report::scip::Package::project( + &relative_to, + self.output.report_scip_project_name.clone(), + self.output.report_scip_project_version.clone(), + )?; + let scip = Arc::new(report::scip::ScipWriter::new( + BufWriter::new(File::create(path)?), + handles, + &relative_to, + project, + version, + )?); + let hook_scip = scip.dupe(); + transaction.set_solutions_hook(Some(Box::new(move |handle, transaction| { + if let Err(e) = hook_scip.write_document(transaction, handle) { + error!( + "while writing SCIP document for `{}`: {e:#}", + handle.module() + ); + } + }))); + transaction.set_evict_after_solutions_hook(true); + Some(scip) + } + None => None, + }; + let type_check_start = Instant::now(); let demand_tree_subscriber = if self.output.report_demand_tree.is_some() { transaction.set_demand_collector(Some(DemandCollector::new())); @@ -1903,6 +1940,7 @@ impl CheckArgs { memory_trace, demand_tree_subscriber, type_check_start, + scip, }) } @@ -1921,6 +1959,7 @@ impl CheckArgs { mut memory_trace, demand_tree_subscriber, type_check_start, + scip, } = run; transaction.set_subscriber(None); @@ -2278,23 +2317,15 @@ impl CheckArgs { )?; } } - if let Some(path) = &self.output.report_scip { - let project = report::scip::Package::project( - &relative_to, - self.output.report_scip_project_name.clone(), - self.output.report_scip_project_version.clone(), - )?; - let mut writer = BufWriter::new(File::create(path)?); - report::scip::write_scip( - &mut writer, - transaction, - handles, - &relative_to, - &project, - version, - ) - .and_then(|()| Ok(writer.flush()?)) - .with_context(|| format!("while writing SCIP index to `{}`", path.display()))?; + if let Some(scip) = scip + && let Some(path) = &self.output.report_scip + { + transaction.set_solutions_hook(None); + transaction.set_evict_after_solutions_hook(false); + let scip = Arc::into_inner(scip).expect("the solutions hook holds the only other copy"); + scip.finish(transaction) + .and_then(|mut writer| Ok(writer.flush()?)) + .with_context(|| format!("while writing SCIP index to `{}`", path.display()))?; } if let Some(pysa_reporter) = transaction.take_pysa_reporter() { report::pysa::write_project_file(&pysa_reporter, transaction, handles, &output_errors)?; diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 573f5fd202..af11b29b0e 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -8,7 +8,6 @@ use std::collections::HashMap; use std::io::Write; use std::iter; -use std::mem; use std::path::Path; use std::path::PathBuf; use std::sync::Arc; @@ -116,10 +115,9 @@ impl Package { } } -/// Writes a SCIP index as a stream of `Index` messages, which a reader merges into one: the -/// metadata, then each document as soon as it is built, in no particular order, then the -/// external symbols. -pub fn write_scip( +/// Writes a SCIP index of `handles` after they have been checked. +#[cfg(test)] +fn write_scip( writer: &mut (impl Write + Send), transaction: &mut Transaction, handles: &[Handle], @@ -127,94 +125,143 @@ pub fn write_scip( project: &Package, version: &str, ) -> anyhow::Result<()> { - let project_root = project_root.absolutize(); - let indexer = Indexer { - transaction: &*transaction, - project_root: &project_root, - project, - definitions: Mutex::new(HashMap::new()), - distributions: Mutex::new(HashMap::new()), - external_symbols: Mutex::new(HashMap::new()), - }; - Index { - metadata: Some(Metadata { - project_root: Uri::from_directory_path(&project_root) - .map(|uri| uri.to_string()) - .unwrap_or_default(), - tool_info: Some(ToolInfo { - name: "pyrefly".to_owned(), - version: version.to_owned(), + let scip = ScipWriter::new(writer, handles, project_root, project.clone(), version)?; + transaction.install(|| { + handles + .par_iter() + .try_for_each(|handle| scip.write_document(transaction, handle)) + })?; + scip.finish(transaction)?; + Ok(()) +} + +/// Writes a SCIP index as a stream of `Index` messages, which a reader merges into one: the +/// metadata, then each document as soon as it is built, in no particular order, then the +/// external symbols. Documents can be written while the files are being checked, from a +/// solutions hook. +pub struct ScipWriter { + writer: Mutex, + handles: SmallSet, + project_root: PathBuf, + project: Package, + definitions: Mutex>>>, + distributions: Mutex>>>, + external_symbols: Mutex>, +} + +impl ScipWriter { + pub fn new( + mut writer: W, + handles: &[Handle], + project_root: &Path, + project: Package, + version: &str, + ) -> anyhow::Result { + let project_root = project_root.absolutize(); + Index { + metadata: Some(Metadata { + project_root: Uri::from_directory_path(&project_root) + .map(|uri| uri.to_string()) + .unwrap_or_default(), + tool_info: Some(ToolInfo { + name: "pyrefly".to_owned(), + version: version.to_owned(), + ..Default::default() + }) + .into(), + text_document_encoding: TextEncoding::UTF8.into(), ..Default::default() }) .into(), - text_document_encoding: TextEncoding::UTF8.into(), ..Default::default() + } + .write_to_writer(&mut writer)?; + Ok(Self { + writer: Mutex::new(writer), + handles: handles.iter().cloned().collect(), + project_root, + project, + definitions: Mutex::new(HashMap::new()), + distributions: Mutex::new(HashMap::new()), + external_symbols: Mutex::new(HashMap::new()), }) - .into(), - ..Default::default() } - .write_to_writer(writer)?; - let documents_writer = Mutex::new(&mut *writer); - transaction.install(|| { - handles.par_iter().try_for_each(|handle| { - let Some(document) = indexer.document(handle) else { - return anyhow::Ok(()); - }; - let bytes = Index { - documents: vec![document], + + fn indexer<'a>(&'a self, transaction: &'a Transaction<'a>) -> Indexer<'a> { + Indexer { + transaction, + project_root: &self.project_root, + project: &self.project, + definitions: &self.definitions, + distributions: &self.distributions, + external_symbols: &self.external_symbols, + } + } + + /// Writes the document of `handle`, if it is one of the indexed files. + pub fn write_document(&self, transaction: &Transaction, handle: &Handle) -> anyhow::Result<()> { + if !self.handles.contains(handle) { + return Ok(()); + } + let Some(document) = self.indexer(transaction).document(handle) else { + return Ok(()); + }; + let bytes = Index { + documents: vec![document], + ..Default::default() + } + .write_to_bytes()?; + self.writer.lock().write_all(&bytes)?; + Ok(()) + } + + /// Writes the external symbols, once every document is written. + pub fn finish(self, transaction: &mut Transaction) -> anyhow::Result { + let Self { + writer, + definitions, + external_symbols, + .. + } = self; + let mut writer = writer.into_inner(); + let mut first_uses: Vec<(String, ExternalUse)> = + external_symbols.into_inner().into_iter().collect(); + // A keyword argument is documented with its parameter's type, which comes from the + // defining module's answers. Those are only kept for modules solved to + // `Require::Everything`, so the symbol is documented at its definition instead. + let dependencies: Vec = definitions + .into_inner() + .into_keys() + .filter(|x| transaction.get_ast(x).is_none()) + .collect(); + transaction.run(&dependencies, Require::Everything, None); + let transaction = &*transaction; + transaction.install(|| { + first_uses.par_iter_mut().for_each(|(_, first_use)| { + if first_use.documentation.is_empty() + && let Some((handle, position)) = &first_use.definition + { + first_use.documentation = documentation(transaction, handle, *position); + } + }) + }); + let mut external_symbols: Vec = first_uses + .into_iter() + .filter(|(_, first_use)| !first_use.documentation.is_empty()) + .map(|(symbol, first_use)| SymbolInformation { + symbol, + documentation: first_use.documentation, ..Default::default() - } - .write_to_bytes()?; - documents_writer.lock().write_all(&bytes)?; - Ok(()) - }) - })?; - let Indexer { - definitions, - distributions, - external_symbols, - .. - } = indexer; - // A keyword argument is documented with its parameter's type, which comes from the defining - // module's answers. Those are only kept for modules solved to `Require::Everything`. - let dependencies: Vec = definitions - .lock() - .keys() - .filter(|x| transaction.get_ast(x).is_none()) - .cloned() - .collect(); - transaction.run(&dependencies, Require::Everything, None); - let indexer = Indexer { - transaction: &*transaction, - project_root: &project_root, - project, - definitions, - distributions, - external_symbols, - }; - let first_uses: Vec<_> = mem::take(&mut *indexer.external_symbols.lock()) - .into_iter() - .collect(); - let mut external_symbols: Vec = transaction.install(|| { - first_uses - .into_par_iter() - .filter_map(|(symbol, first_use)| { - let documentation = indexer.documentation(&first_use.handle, first_use.position); - (!documentation.is_empty()).then_some(SymbolInformation { - symbol, - documentation, - ..Default::default() - }) }) - .collect() - }); - external_symbols.sort_by(|a, b| a.symbol.cmp(&b.symbol)); - Index { - external_symbols, - ..Default::default() + .collect(); + external_symbols.sort_by(|a, b| a.symbol.cmp(&b.symbol)); + Index { + external_symbols, + ..Default::default() + } + .write_to_writer(&mut writer)?; + Ok(writer) } - .write_to_writer(writer)?; - Ok(()) } /// Top-level names whose kind in the bundled stubs differs from how other indexers name them: @@ -230,19 +277,25 @@ struct Indexer<'a> { project_root: &'a Path, project: &'a Package, /// The definitions of every module seen so far, keyed by the range of the defining identifier. - definitions: Mutex>>>, + definitions: &'a Mutex>>>, /// The installed distributions of every site-packages directory seen so far. - distributions: Mutex>>>, + distributions: &'a Mutex>>>, /// The first use, in path and position order, of every symbol defined outside the project and /// referenced so far. The symbol is documented with the hover there, so its documentation /// doesn't depend on the order documents are built in. - external_symbols: Mutex>, + external_symbols: &'a Mutex>, } +/// For a symbol defined outside the project, where to document it when the hover at its use has +/// nothing. +type External = Option>; + struct ExternalUse { relative_path: PathBuf, position: TextSize, - handle: Handle, + documentation: Vec, + /// Where the symbol is defined, to document it there when the hover at the use has nothing. + definition: Option<(Handle, TextSize)>, } impl Indexer<'_> { @@ -311,25 +364,29 @@ impl Indexer<'_> { for x in uses { let symbol = match (self.reference(handle, &module, &x), x.kind) { (Some((symbol, external)), _) => { - if external { + if let Some(definition) = external { let position = use_position(&x); - let mut external_symbols = self.external_symbols.lock(); - let first_use = - external_symbols - .entry(symbol.clone()) - .or_insert_with(|| ExternalUse { - relative_path: relative_path.to_owned(), - position, - handle: handle.dupe(), - }); - if (relative_path, position) - < (first_use.relative_path.as_path(), first_use.position) - { - *first_use = ExternalUse { - relative_path: relative_path.to_owned(), - position, - handle: handle.dupe(), - }; + let is_first = |external_symbols: &HashMap| { + external_symbols.get(&symbol).is_none_or(|first_use| { + (relative_path, position) + < (first_use.relative_path.as_path(), first_use.position) + }) + }; + // Documented here, while this file's answers are still in memory. + if is_first(&self.external_symbols.lock()) { + let documentation = documentation(self.transaction, handle, position); + let mut external_symbols = self.external_symbols.lock(); + if is_first(&external_symbols) { + external_symbols.insert( + symbol.clone(), + ExternalUse { + relative_path: relative_path.to_owned(), + position, + documentation, + definition, + }, + ); + } } } symbol @@ -366,7 +423,7 @@ impl Indexer<'_> { for (range, symbol) in definitions.iter() { let symbol = format_scip_symbol(self.project, symbol); if !symbols.contains_key(&symbol) { - let documentation = self.documentation(handle, range.start()); + let documentation = documentation(self.transaction, handle, range.start()); symbols.insert( symbol.clone(), SymbolInformation { @@ -473,29 +530,9 @@ impl Indexer<'_> { implementations } - /// The hover text at a position, as SCIP documentation. - fn documentation(&self, handle: &Handle, position: TextSize) -> Vec { - let options = HoverOptions { - show_go_to_links: false, - verbosity_level: 0, - field_docstring_at_definition: true, - }; - match get_hover_with_verbosity(self.transaction, handle, position, options) { - Some(HoverResult { - hover: - Hover { - contents: Contents::MarkupContent(markup), - .. - }, - .. - }) => vec![markup.value], - _ => Vec::new(), - } - } - - /// The symbol a use refers to, and whether it is defined outside the project, or `None` if - /// it has no definition we can name. - fn reference(&self, handle: &Handle, module: &Module, x: &Use) -> Option<(String, bool)> { + /// The symbol a use refers to, and where it is defined if that is outside the project, or + /// `None` if it has no definition we can name. + fn reference(&self, handle: &Handle, module: &Module, x: &Use) -> Option<(String, External)> { let preference = preference(); let definition = match x.kind { UseKind::Name => { @@ -513,7 +550,10 @@ impl Indexer<'_> { descriptor("builtins", Suffix::Namespace), descriptor(&x.identifier.id, Suffix::Type), ]; - return Some((global_symbol(&stdlib_package(handle), descriptors), true)); + return Some(( + global_symbol(&stdlib_package(handle), descriptors), + Some(None), + )); } definition } @@ -535,18 +575,29 @@ impl Indexer<'_> { handle: &Handle, module: &Module, definition: FindDefinitionItemWithDocstring, - ) -> Option<(String, bool)> { - let external = !definition + ) -> Option<(String, External)> { + let external = (!definition .module .path() .as_path() - .starts_with(self.project_root); + .starts_with(self.project_root)) + .then(|| { + let defining = Handle::new( + definition.module.name(), + definition.module.path().dupe(), + handle.sys_info().dupe(), + ); + Some((defining, definition.definition_range.start())) + }); let package = self.package(&definition.module, handle); // A submodule accessed as an attribute resolves to the empty range at the module's start. if definition.metadata.symbol_kind() == Some(SymbolKind::Module) || definition.definition_range.is_empty() { - return Some((module_symbol(&package, definition.module.name()), external)); + return Some(( + module_symbol(&package, definition.module.name()), + external.map(|_| None), + )); } let symbols = self.definitions(&definition.module, handle.sys_info().dupe())?; match symbols.get(&definition.definition_range)? { @@ -554,7 +605,7 @@ impl Indexer<'_> { Some((global_symbol(&package, descriptors.clone()), external)) } ScipSymbol::Local(id) => { - (definition.module.path() == module.path()).then(|| (format!("local {id}"), false)) + (definition.module.path() == module.path()).then(|| (format!("local {id}"), None)) } } } @@ -628,6 +679,26 @@ impl Indexer<'_> { } } +/// The hover text at a position, as SCIP documentation. +fn documentation(transaction: &Transaction, handle: &Handle, position: TextSize) -> Vec { + let options = HoverOptions { + show_go_to_links: false, + verbosity_level: 0, + field_docstring_at_definition: true, + }; + match get_hover_with_verbosity(transaction, handle, position, options) { + Some(HoverResult { + hover: + Hover { + contents: Contents::MarkupContent(markup), + .. + }, + .. + }) => vec![markup.value], + _ => Vec::new(), + } +} + fn stdlib_package(handle: &Handle) -> Package { let version = handle.sys_info().version(); Package { diff --git a/pyrefly/lib/state/state.rs b/pyrefly/lib/state/state.rs index 8c2dea2476..d79dfaf810 100644 --- a/pyrefly/lib/state/state.rs +++ b/pyrefly/lib/state/state.rs @@ -715,6 +715,9 @@ pub(crate) struct TransactionData<'a> { cinderx_reporter: Option>, /// When set, called per solved module while its answers are still live (before eviction). solutions_hook: Option>, + /// When set, the answers of a module are evicted once the solutions hook has seen it, + /// whatever its require level. + evict_after_solutions_hook: bool, } impl<'a> TransactionData<'a> { @@ -867,6 +870,12 @@ impl<'a> Transaction<'a> { self.data.solutions_hook = hook; } + /// Evict each module's answers once the solutions hook has seen it, so a hook that needs + /// them at `Require::Everything` doesn't keep them for the rest of the run. + pub fn set_evict_after_solutions_hook(&mut self, evict: bool) { + self.data.evict_after_solutions_hook = evict; + } + /// Take the pysa reporter out of the transaction, consuming ownership. pub fn take_pysa_reporter(&mut self) -> Option> { self.data.pysa_reporter.take() @@ -1684,7 +1693,7 @@ impl<'a> Transaction<'a> { if let Some(hook) = &self.data.solutions_hook { hook(&module_data.handle, self); } - if !require.keep_answers() { + if !require.keep_answers() || self.data.evict_after_solutions_hook { // From now on we can use the solutions directly, so evict the answers. post.evict_answers(); } @@ -3615,6 +3624,7 @@ impl State { pysa_reporter: None, cinderx_reporter: None, solutions_hook: None, + evict_after_solutions_hook: false, }, } } @@ -3719,6 +3729,7 @@ impl State { pysa_reporter: _, cinderx_reporter: _, solutions_hook: _, + evict_after_solutions_hook: _, }, }, committing_transaction_guard, From 19a180f6fef1b330e2e8b401773c84aceb53b797 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Tue, 3 Nov 2026 08:07:47 -0500 Subject: [PATCH 19/21] fix more bugs and format --- pyrefly/lib/report/scip.rs | 97 +++++++++++++++---- .../snapshots/input/lambda_walrus/main.py | 1 - .../input/nonlocal_skips_scope/main.py | 1 - .../input/project_site_packages/main.py | 1 - .../snapshots/input/type_parameters/main.py | 1 - .../snapshots/output/lambda_walrus/main.py | 5 +- .../output/nonlocal_skips_scope/main.py | 5 +- .../output/project_site_packages/main.py | 7 +- .../snapshots/output/type_parameters/main.py | 6 +- pyrefly/lib/report/scip/symbols.rs | 67 ++++++++++--- pyrefly/lib/state/lsp.rs | 22 ++++- ruff.toml | 1 + 12 files changed, 164 insertions(+), 50 deletions(-) diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 9c1664098a..fec7de9048 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -36,6 +36,7 @@ use rayon::prelude::*; use ruff_python_ast::Expr; use ruff_python_ast::ExprContext; use ruff_python_ast::Identifier; +use ruff_python_ast::ModModule; use ruff_python_ast::Stmt; use ruff_python_ast::StmtClassDef; use ruff_python_ast::name::Name; @@ -66,8 +67,10 @@ use crate::lsp::wasm::hover::get_hover_with_verbosity; use crate::report::scip::symbols::ScipSymbol; use crate::report::scip::symbols::definitions; use crate::report::scip::symbols::descriptor; +use crate::state::lsp::CalleeKind; use crate::state::lsp::FindDefinitionItemWithDocstring; use crate::state::lsp::FindPreference; +use crate::state::lsp::callee_kind_from_call; use crate::state::require::Require; use crate::state::state::Transaction; use crate::types::globals::ImplicitGlobal; @@ -145,6 +148,7 @@ pub struct ScipWriter { project_root: PathBuf, project: Package, definitions: Mutex>>>, + asts: Mutex>>, distributions: Mutex>>>, external_symbols: Mutex>, } @@ -182,6 +186,7 @@ impl ScipWriter { project_root, project, definitions: Mutex::new(HashMap::new()), + asts: Mutex::new(HashMap::new()), distributions: Mutex::new(HashMap::new()), external_symbols: Mutex::new(HashMap::new()), }) @@ -193,6 +198,7 @@ impl ScipWriter { project_root: &self.project_root, project: &self.project, definitions: &self.definitions, + asts: &self.asts, distributions: &self.distributions, external_symbols: &self.external_symbols, } @@ -278,6 +284,8 @@ struct Indexer<'a> { project: &'a Package, /// The definitions of every module seen so far, keyed by the range of the defining identifier. definitions: &'a Mutex>>>, + /// The parsed AST of every dependency seen so far, since only the indexed files keep theirs. + asts: &'a Mutex>>, /// The installed distributions of every site-packages directory seen so far. distributions: &'a Mutex>>>, /// The first use, in path and position order, of every symbol defined outside the project and @@ -362,7 +370,7 @@ impl Indexer<'_> { uses.retain(|x| !definitions.contains_key(&x.range)); uses.sort_by_key(|x| x.range.start()); for x in uses { - let symbol = match (self.reference(handle, &module, &x), x.kind) { + let symbol = match (self.reference(handle, &module, &x), &x.kind) { (Some((symbol, external)), _) => { if let Some(definition) = external { let position = use_position(&x); @@ -396,7 +404,7 @@ impl Indexer<'_> { format!("local {}", unresolved.entry(x.identifier.id).or_insert(id)) } // An unresolved attribute or keyword isn't a variable, so it gets no local. - (None, UseKind::Attribute | UseKind::Keyword) => continue, + (None, UseKind::Attribute | UseKind::Keyword(_)) => continue, }; occurrences.push(Occurrence { range: scip_range(&module, x.range), @@ -534,7 +542,7 @@ impl Indexer<'_> { /// `None` if it has no definition we can name. fn reference(&self, handle: &Handle, module: &Module, x: &Use) -> Option<(String, External)> { let preference = preference(); - let definition = match x.kind { + let definition = match &x.kind { UseKind::Name => { let definition = self .transaction @@ -557,7 +565,36 @@ impl Indexer<'_> { } definition } - UseKind::Import | UseKind::Attribute | UseKind::Keyword => { + UseKind::Keyword(callee_kind) => { + // Dependencies keep no AST, so the callee's module comes from our cache rather + // than being parsed again for every keyword. + let definition = self + .transaction + .find_definition_for_keyword_argument_with_ast( + handle, + &x.identifier, + callee_kind, + preference, + |module| { + let defining = Handle::new( + module.name(), + module.path().dupe(), + handle.sys_info().dupe(), + ); + self.ast(module, &defining) + }, + ) + .into_iter() + .next()?; + Some(FindDefinitionItemWithDocstring { + metadata: definition.metadata, + definition_range: definition.definition_range, + module: definition.module, + docstring_range: None, + display_name: Some(x.identifier.id.to_string()), + }) + } + UseKind::Import | UseKind::Attribute => { let definitions = self .transaction .find_definition(handle, use_position(x), preference) @@ -576,12 +613,7 @@ impl Indexer<'_> { module: &Module, definition: FindDefinitionItemWithDocstring, ) -> Option<(String, External)> { - let external = (!definition - .module - .path() - .as_path() - .starts_with(self.project_root)) - .then(|| { + let external = (!self.in_project(&definition.module, handle)).then(|| { let defining = Handle::new( definition.module.name(), definition.module.path().dupe(), @@ -619,10 +651,7 @@ impl Indexer<'_> { if let Some(definitions) = self.definitions.lock().get(&handle) { return Some(definitions.dupe()); } - // Only the indexed files keep their AST, so a dependency is parsed again. - let ast = self.transaction.get_ast(&handle).unwrap_or_else(|| { - Arc::new(Ast::parse(module.contents(), module.source_type()).0) - }); + let ast = self.ast(module, &handle); let mut table: SmallMap<_, _> = definitions(module.name(), &ast).into_iter().collect(); for (module_name, name, suffix) in SPECIAL_FORMS { if module.name().as_str() != *module_name { @@ -642,11 +671,40 @@ impl Indexer<'_> { Some(table) } + /// The AST of `module`, parsed again if it is a dependency. + fn ast(&self, module: &Module, handle: &Handle) -> Arc { + if let Some(ast) = self.transaction.get_ast(handle) { + return ast; + } + if let Some(ast) = self.asts.lock().get(handle) { + return ast.dupe(); + } + let ast = Arc::new(Ast::parse(module.contents(), module.source_type()).0); + self.asts.lock().insert(handle.clone(), ast.dupe()); + ast + } + + /// Whether a module referenced from `handle` belongs to the project: it is under the project + /// root, and not in a site-packages directory there, such as a virtual environment's. + fn in_project(&self, module: &Module, handle: &Handle) -> bool { + let path = module.path().as_path(); + path.starts_with(self.project_root) + && !self + .transaction + .get_config(handle) + .iter() + .flat_map(|config| config.site_package_path()) + .any(|root| path.starts_with(root)) + } + /// The package of a module referenced from `handle`, whose site-packages are searched for /// installed distributions. fn package(&self, module: &Module, handle: &Handle) -> Package { - let bundled = matches!(module.path().details(), ModulePathDetails::BundledTypeshed(_)); - if !bundled && module.path().as_path().starts_with(self.project_root) { + let bundled = matches!( + module.path().details(), + ModulePathDetails::BundledTypeshed(_) + ); + if !bundled && self.in_project(module, handle) { return self.project.clone(); } // Match by top-level module, so that bundled stubs get the installed package too, even for @@ -762,15 +820,14 @@ struct Use { kind: UseKind, } -#[derive(Clone, Copy)] enum UseKind { /// A name in an expression. Name, /// A module or name in an import statement. Import, Attribute, - /// The name of a keyword argument in a call. - Keyword, + /// The name of a keyword argument in a call to the callee. + Keyword(CalleeKind), } /// Collects every name that is read or deleted, every attribute, and every keyword argument. @@ -790,7 +847,7 @@ fn collect_expr_uses(x: &Expr, uses: &mut Vec) { Expr::Call(call) => { for keyword in call.arguments.keywords.iter() { if let Some(arg) = &keyword.arg { - push(arg.clone(), UseKind::Keyword); + push(arg.clone(), UseKind::Keyword(callee_kind_from_call(call))); } } } diff --git a/pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py b/pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py index 3e369c1d4b..f0892200ef 100644 --- a/pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py +++ b/pyrefly/lib/report/scip/snapshots/input/lambda_walrus/main.py @@ -1,4 +1,3 @@ -# BUG: These lambda-local bindings share a module-level symbol. # Each `hidden` should have a distinct local symbol. first = lambda: (hidden := 1) second = lambda: (hidden := 2) diff --git a/pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py b/pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py index 89651d6e47..a3aebb3db3 100644 --- a/pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py +++ b/pyrefly/lib/report/scip/snapshots/input/nonlocal_skips_scope/main.py @@ -1,4 +1,3 @@ -# BUG: The nonlocal assignment has a different symbol from the outer binding. # All occurrences of `x` should share the same symbol, skipping `middle`. def outer(): x = 0 diff --git a/pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py index 88ca166ada..34ed8fce6b 100644 --- a/pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py +++ b/pyrefly/lib/report/scip/snapshots/input/project_site_packages/main.py @@ -1,4 +1,3 @@ -# BUG: This dependency inside the project root gets the project's package identity. # Its symbols should belong to review-dependency 2.0.0. from review_dependency import Widget diff --git a/pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py b/pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py index 974791f214..624e24a544 100644 --- a/pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py +++ b/pyrefly/lib/report/scip/snapshots/input/type_parameters/main.py @@ -1,4 +1,3 @@ -# BUG: The type parameter has references but no definition occurrence. # The declaration of `T` and both annotations should share a defined symbol. def identity[T](x: T) -> T: return x diff --git a/pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py b/pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py index f0ca866e25..75445d76e7 100644 --- a/pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/lambda_walrus/main.py @@ -1,11 +1,10 @@ # < definition scip-python python snapshot-util 0.1 main/__init__: -# BUG: These lambda-local bindings share a module-level symbol. # Each `hidden` should have a distinct local symbol. first = lambda: (hidden := 1) #^^^^ definition snapshot-util 0.1 main/first. -# ^^^^^^ definition snapshot-util 0.1 main/hidden. +# ^^^^^^ definition local 0 second = lambda: (hidden := 2) #^^^^^ definition snapshot-util 0.1 main/second. -# ^^^^^^ definition snapshot-util 0.1 main/hidden. +# ^^^^^^ definition local 1 diff --git a/pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py b/pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py index 596736f52c..e5ec1d8c5b 100644 --- a/pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/nonlocal_skips_scope/main.py @@ -1,6 +1,5 @@ # < definition scip-python python snapshot-util 0.1 main/__init__: -# BUG: The nonlocal assignment has a different symbol from the outer binding. # All occurrences of `x` should share the same symbol, skipping `middle`. def outer(): # ^^^^^ definition snapshot-util 0.1 main/outer(). @@ -13,9 +12,9 @@ def inner(): # ^^^^^ definition snapshot-util 0.1 main/outer().middle().inner(). nonlocal x x = 1 -# ^ definition local 1 +# ^ reference local 0 return x -# ^ reference local 1 +# ^ reference local 0 return inner # ^^^^^ reference snapshot-util 0.1 main/outer().middle().inner(). diff --git a/pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py b/pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py index ce9b1ef210..0982800329 100644 --- a/pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/project_site_packages/main.py @@ -1,12 +1,11 @@ # < definition scip-python python snapshot-util 0.1 main/__init__: -# BUG: This dependency inside the project root gets the project's package identity. # Its symbols should belong to review-dependency 2.0.0. from review_dependency import Widget -# ^^^^^^^^^^^^^^^^^ reference snapshot-util 0.1 review_dependency/__init__: -# ^^^^^^ reference snapshot-util 0.1 review_dependency/Widget# +# ^^^^^^^^^^^^^^^^^ reference review-dependency 2.0.0 review_dependency/__init__: +# ^^^^^^ reference review-dependency 2.0.0 review_dependency/Widget# widget = Widget() #^^^^^ definition snapshot-util 0.1 main/widget. -# ^^^^^^ reference snapshot-util 0.1 review_dependency/Widget# +# ^^^^^^ reference review-dependency 2.0.0 review_dependency/Widget# diff --git a/pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py b/pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py index cb43213062..68626af508 100644 --- a/pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py +++ b/pyrefly/lib/report/scip/snapshots/output/type_parameters/main.py @@ -1,12 +1,12 @@ # < definition scip-python python snapshot-util 0.1 main/__init__: -# BUG: The type parameter has references but no definition occurrence. # The declaration of `T` and both annotations should share a defined symbol. def identity[T](x: T) -> T: # ^^^^^^^^ definition snapshot-util 0.1 main/identity(). +# ^ definition snapshot-util 0.1 main/identity().[T] # ^ definition snapshot-util 0.1 main/identity().(x) -# ^ reference local 0 -# ^ reference local 0 +# ^ reference snapshot-util 0.1 main/identity().[T] +# ^ reference snapshot-util 0.1 main/identity().[T] return x # ^ reference snapshot-util 0.1 main/identity().(x) diff --git a/pyrefly/lib/report/scip/symbols.rs b/pyrefly/lib/report/scip/symbols.rs index b9002fb230..5b2aa09b30 100644 --- a/pyrefly/lib/report/scip/symbols.rs +++ b/pyrefly/lib/report/scip/symbols.rs @@ -19,12 +19,14 @@ use ruff_python_ast::ModModule; use ruff_python_ast::Parameters; use ruff_python_ast::Pattern; use ruff_python_ast::Stmt; +use ruff_python_ast::TypeParams; use ruff_python_ast::name::Name; use ruff_python_ast::visitor::source_order::SourceOrderVisitor; use ruff_python_ast::visitor::source_order::walk_except_handler; use ruff_python_ast::visitor::source_order::walk_expr; use ruff_python_ast::visitor::source_order::walk_pattern; use ruff_python_ast::visitor::source_order::walk_stmt; +use ruff_python_ast::visitor::source_order::walk_type_param; use ruff_text_size::TextRange; use scip::types::Descriptor; use scip::types::descriptor::Suffix; @@ -71,8 +73,10 @@ enum ScopeKind { globals: SmallSet, nonlocals: SmallSet, }, - /// A comprehension or lambda: every name it binds is a local. - Anonymous, + /// Every name a comprehension binds is a local, except a walrus target. + Comprehension, + /// Every name a lambda binds is a local. + Lambda, } struct Scope { @@ -150,15 +154,23 @@ impl Builder { ScipSymbol::Global(child(&self.module_path, name, Suffix::Term)) } ScopeKind::Function { nonlocals, .. } if nonlocals.contains(name) => { - match (0..index) + // The nearest enclosing function that binds the name, or else the nearest one. + let functions: Vec = (0..index) .rev() - .find(|i| matches!(self.scopes[*i].kind, ScopeKind::Function { .. })) + .filter(|i| matches!(self.scopes[*i].kind, ScopeKind::Function { .. })) + .collect(); + match functions + .iter() + .find(|i| self.scopes[**i].names.contains_key(name)) + .or(functions.first()) { - Some(enclosing) => self.binding(enclosing, name), + Some(enclosing) => self.binding(*enclosing, name), None => self.new_local(), } } - ScopeKind::Function { .. } | ScopeKind::Anonymous => self.new_local(), + ScopeKind::Function { .. } | ScopeKind::Comprehension | ScopeKind::Lambda => { + self.new_local() + } }; self.scopes[index] .names @@ -171,7 +183,7 @@ impl Builder { fn bind(&mut self, id: &Identifier, walrus: bool) { let mut index = self.scopes.len() - 1; if walrus { - while index > 0 && matches!(self.scopes[index].kind, ScopeKind::Anonymous) { + while index > 0 && matches!(self.scopes[index].kind, ScopeKind::Comprehension) { index -= 1; } } @@ -206,20 +218,36 @@ impl Builder { for parameter in parameters.iter() { let id = parameter.name(); let symbol = match &self.scope().kind { - ScopeKind::Anonymous => self.new_local(), + ScopeKind::Comprehension | ScopeKind::Lambda => self.new_local(), _ => ScipSymbol::Global(child(&self.scope().path, &id.id, Suffix::Parameter)), }; self.define_in_scope(id, symbol); } } + /// Type parameters belong to the function, class or type alias that declares them. + fn bind_type_params(&mut self, type_params: Option<&TypeParams>, owner: &ScipSymbol) { + for type_param in type_params.iter().flat_map(|x| &x.type_params) { + let symbol = match owner { + ScipSymbol::Global(path) => { + ScipSymbol::Global(child(path, &type_param.name().id, Suffix::TypeParameter)) + } + ScipSymbol::Local(_) => self.new_local(), + }; + self.define(type_param.name(), symbol); + walk_type_param(self, type_param); + } + } + fn visit_comprehensions(&mut self, generators: &[Comprehension], elements: &[&Expr]) { // The first iterable is evaluated in the enclosing scope. if let Some(first) = generators.first() { self.visit_expr(&first.iter); } - self.scopes - .push(Scope::new(ScopeKind::Anonymous, self.scope().path.clone())); + self.scopes.push(Scope::new( + ScopeKind::Comprehension, + self.scope().path.clone(), + )); for (i, generator) in generators.iter().enumerate() { if i > 0 { self.visit_expr(&generator.iter); @@ -248,6 +276,10 @@ impl<'a> SourceOrderVisitor<'a> for Builder { self.visit_expr(returns); } let path = self.declare(&function.name, Suffix::Method); + self.bind_type_params( + function.type_params.as_deref(), + &ScipSymbol::Global(path.clone()), + ); let receiver = match &self.scope().kind { ScopeKind::Class { .. } => function .parameters @@ -277,6 +309,10 @@ impl<'a> SourceOrderVisitor<'a> for Builder { self.visit_arguments(arguments); } let path = self.declare(&class.name, Suffix::Type); + self.bind_type_params( + class.type_params.as_deref(), + &ScipSymbol::Global(path.clone()), + ); let mut attributes = SmallSet::new(); for stmt in &class.body { match stmt { @@ -302,6 +338,15 @@ impl<'a> SourceOrderVisitor<'a> for Builder { self.visit_body(&class.body); self.scopes.pop(); } + Stmt::TypeAlias(alias) => { + self.visit_expr(&alias.name); + if let Expr::Name(name) = &*alias.name + && let Some(owner) = self.scope().names.get(&name.id).cloned() + { + self.bind_type_params(alias.type_params.as_deref(), &owner); + } + self.visit_expr(&alias.value); + } Stmt::AugAssign(assign) => { if !matches!(&*assign.target, Expr::Name(_)) { self.visit_expr(&assign.target); @@ -340,7 +385,7 @@ impl<'a> SourceOrderVisitor<'a> for Builder { self.visit_parameters(parameters); } self.scopes - .push(Scope::new(ScopeKind::Anonymous, self.scope().path.clone())); + .push(Scope::new(ScopeKind::Lambda, self.scope().path.clone())); if let Some(parameters) = &lambda.parameters { self.bind_parameters(parameters); } diff --git a/pyrefly/lib/state/lsp.rs b/pyrefly/lib/state/lsp.rs index a1a27905dd..d6c98abee0 100644 --- a/pyrefly/lib/state/lsp.rs +++ b/pyrefly/lib/state/lsp.rs @@ -131,7 +131,7 @@ struct OperatorDunder { range: TextRange, } -fn callee_kind_from_call(call: &ExprCall) -> CalleeKind { +pub(crate) fn callee_kind_from_call(call: &ExprCall) -> CalleeKind { match call.func.as_ref() { Expr::Name(name) => CalleeKind::Function(Ast::expr_name_identifier(name.clone())), Expr::Attribute(attr) => CalleeKind::Method(attr.value.range(), attr.attr.clone()), @@ -2733,6 +2733,24 @@ impl<'a> Transaction<'a> { identifier: &Identifier, callee_kind: &CalleeKind, preference: FindPreference, + ) -> Vec { + self.find_definition_for_keyword_argument_with_ast( + handle, + identifier, + callee_kind, + preference, + |module| self.get_ast_or_parse_module(handle, module), + ) + } + + /// Like `find_definition_for_keyword_argument`, but gets the callee module's AST from `get_ast`. + pub(crate) fn find_definition_for_keyword_argument_with_ast( + &self, + handle: &Handle, + identifier: &Identifier, + callee_kind: &CalleeKind, + preference: FindPreference, + get_ast: impl Fn(&ModuleInfo) -> Arc, ) -> Vec { // NOTE(grievejia): There might be a better way to compute this that doesn't require 2 containing node // traversal, once we gain access to the callee function def from callee_kind directly. @@ -2752,7 +2770,7 @@ impl<'a> Transaction<'a> { let mut results: Vec = Vec::with_capacity(location_count); for (module_info, ranges) in modules_to_ranges.into_iter() { - let ast = self.get_ast_or_parse_module(handle, &module_info); + let ast = get_ast(&module_info); for range in ranges.into_iter() { let (metadata, definition_range) = if let Some((definition_range, metadata)) = self diff --git a/ruff.toml b/ruff.toml index 1fc686fb8a..7d891a40c0 100644 --- a/ruff.toml +++ b/ruff.toml @@ -5,6 +5,7 @@ extend-exclude = [ "conformance/**", "crates/pyrefly_bundled/**", "crates/tsp_types/**", + "pyrefly/lib/report/scip/snapshots/**", "pyrefly/lib/test/**", "test/**", "website/**", From 23d9053eeaa13f2bd1c4c930ac60e5aebfb658dd Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Tue, 3 Nov 2026 22:47:09 +0000 Subject: [PATCH 20/21] Name the SCIP indexer scip-pyrefly SCIP indexes name the tool that produced them in metadata.tool_info.name, and indexers conventionally use a scip- prefix (scip-python, scip-java, scip-typescript). Writing scip-pyrefly makes the producer clear to SCIP consumers alongside other indexers. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_017Bffbuhqr9Q4j6RVYDgryu --- pyrefly/lib/commands/check.rs | 2 +- pyrefly/lib/report/scip.rs | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyrefly/lib/commands/check.rs b/pyrefly/lib/commands/check.rs index 55e35365c9..52e41ade02 100644 --- a/pyrefly/lib/commands/check.rs +++ b/pyrefly/lib/commands/check.rs @@ -2655,7 +2655,7 @@ def go(w: Widget) -> int: "scip-python python demo . main/__init__:" ); let metadata = index.metadata.unwrap(); - assert_eq!(metadata.tool_info.name, "pyrefly"); + assert_eq!(metadata.tool_info.name, "scip-pyrefly"); assert_eq!(metadata.tool_info.version, "test"); assert!( metadata.project_root.starts_with("file://") diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index fec7de9048..2ee4206418 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -168,7 +168,7 @@ impl ScipWriter { .map(|uri| uri.to_string()) .unwrap_or_default(), tool_info: Some(ToolInfo { - name: "pyrefly".to_owned(), + name: "scip-pyrefly".to_owned(), version: version.to_owned(), ..Default::default() }) From 98aedc68f0d77a4b371cd2f1ce844ce49709a315 Mon Sep 17 00:00:00 2001 From: Danny Yang Date: Wed, 4 Nov 2026 00:02:43 -0500 Subject: [PATCH 21/21] fix line endings for windows --- .gitattributes | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitattributes b/.gitattributes index 09e6018f85..2dab05338d 100644 --- a/.gitattributes +++ b/.gitattributes @@ -1 +1,2 @@ *.mdx text eol=lf +pyrefly/lib/report/scip/snapshots/** text eol=lf