diff --git a/pyrefly/lib/report/scip.rs b/pyrefly/lib/report/scip.rs index 0e30976fc5..1172ba994c 100644 --- a/pyrefly/lib/report/scip.rs +++ b/pyrefly/lib/report/scip.rs @@ -122,12 +122,13 @@ impl Package { } /// Writes a SCIP index as a stream of `Index` messages, which a reader merges into one: the -/// metadata, then each document as soon as it is built, in no particular order, then the -/// external symbols. Documents can be written while the files are being checked, from a -/// solutions hook. +/// metadata, then each document in path order, then the external symbols. Documents can be +/// built while the files are being checked, from a solutions hook, and are kept serialized +/// until they are written. pub struct ScipWriter { writer: Mutex, handles: SmallSet, + documents: Mutex)>>, project_root: PathBuf, project: Package, definitions: Mutex>>>, @@ -171,6 +172,7 @@ impl ScipWriter { Ok(Self { writer: Mutex::new(writer), handles: handles.iter().cloned().collect(), + documents: Mutex::new(Vec::new()), project_root, project, definitions: Mutex::new(HashMap::new()), @@ -211,7 +213,7 @@ impl ScipWriter { } } - /// Writes the document of `handle`, if it is one of the indexed files. + /// Builds the document of `handle`, if it is one of the indexed files. pub fn write_document(&self, transaction: &Transaction, handle: &Handle) -> anyhow::Result<()> { if !self.handles.contains(handle) { return Ok(()); @@ -219,22 +221,24 @@ impl ScipWriter { let Some(document) = self.indexer(transaction).document(handle) else { return Ok(()); }; + let path = document.relative_path.clone(); let bytes = Index { documents: vec![document], ..Default::default() } .write_to_bytes()?; - self.writer.lock().write_all(&bytes)?; + self.documents.lock().push((path, bytes)); Ok(()) } - /// Detaches the solutions hook and writes external symbols after every document is written. + /// Detaches the solutions hook, then writes the documents and the external symbols. pub fn finish(self: Arc, transaction: &mut Transaction) -> anyhow::Result { transaction.set_solutions_hook(None); transaction.set_evict_answers_after_scip_hook(false); let scip = Arc::into_inner(self).expect("the solutions hook holds the only other copy"); let Self { writer, + documents, external_symbols, definitions, asts, @@ -243,6 +247,11 @@ impl ScipWriter { } = scip; drop((definitions, asts, distributions)); let mut writer = writer.into_inner(); + let mut documents = documents.into_inner(); + documents.sort_unstable_by(|a, b| a.0.cmp(&b.0)); + for (_, bytes) in documents { + writer.write_all(&bytes)?; + } let mut first_uses: Vec<(String, ExternalUse)> = external_symbols.into_inner().into_iter().collect(); // Most external symbols are documented at their first use. When that hover has no diff --git a/website/docs/scip.mdx b/website/docs/scip.mdx index 4be34ccd80..9bd43c6fda 100644 --- a/website/docs/scip.mdx +++ b/website/docs/scip.mdx @@ -1,5 +1,5 @@ --- -title: SCIP Indexing +title: Indexing description: Generate a SCIP code-navigation index with Pyrefly. --- @@ -10,7 +10,11 @@ description: Generate a SCIP code-navigation index with Pyrefly. * LICENSE file in the root directory of this source tree. */} -# SCIP Indexing +# Indexing + +:::warning Experimental +This feature is experimental and under active development. The output and command-line options may change in future releases. +::: Pyrefly can generate a [SCIP](https://github.com/scip-code/scip) index for code-navigation tools that consume this format. The index records definitions, references, hover documentation, @@ -22,12 +26,15 @@ From the project root, run: pyrefly check --report-scip index.scip ``` -Pyrefly uses its normal configuration and import resolution. Install the project's dependencies -in the Python environment you use for indexing so that references to external packages can be -resolved. The index contains documents for the files being checked, with paths relative to the +Pyrefly uses its normal configuration and import resolution. The index contains documents for the files being checked, with paths relative to the current directory. Use `--relative-to path/to/project` to select a different project root. In either case, only files under the project root become documents. +Type errors are still reported normally and can make the command return a nonzero exit status +even when it writes an index. + +Symbols in statically unreachable branches (based on platform/Python version or the `@no_type_check` decorator) are not analyzed. + ## Package identity The project's symbols use the package name and version from `[project]` in the root's @@ -39,20 +46,21 @@ pyrefly check --report-scip index.scip \ --report-scip-project-version 1.0.0 ``` -A missing name or version is represented as `.` in SCIP symbols. External package identities -come from installed distribution metadata; bundled standard-library stubs use the Python version +External package identities come from installed distribution metadata, and bundled standard-library stubs use the Python version configured for the project. ## Compatibility -The output is a binary SCIP index, not JSON. It uses the `scip-python` symbol scheme for -compatibility with existing consumers, but its output is not identical to Sourcegraph's -[scip-python](https://github.com/sourcegraph/scip-python) indexer. +:::tip +If your tools need [LSIF](https://microsoft.github.io/language-server-protocol/specifications/lsif/0.6.0/specification/) +instead, convert the index with the [`scip` CLI](https://github.com/scip-code/scip): -Type errors are still reported normally and can make the command return a nonzero exit status -even when it writes an index. +```sh +scip convert --from index.scip --to dump.lsif +``` + +::: + +Pyrefly's output is close to Sourcegraph's [scip-python](https://github.com/sourcegraph/scip-python) +indexer and uses the same symbol naming format, but outputs are not identical. -SCIP reporting supports one-shot checks, not `--watch`: incremental checks do not rewrite -unchanged documents. It also conflicts with `--debug-info`, `--report-binding-memory`, -`--report-glean`, `--report-pysa`, `--report-trace`, `--report-demand-tree`, and `--report-cinderx`: -those reports need state that streaming SCIP releases. diff --git a/website/sidebars.ts b/website/sidebars.ts index e947b9152e..623214d39d 100644 --- a/website/sidebars.ts +++ b/website/sidebars.ts @@ -192,7 +192,7 @@ let docsSidebar = [ { type: 'doc' as const, id: 'scip', - label: 'SCIP Indexing', + label: 'Indexing', }, { type: 'doc' as const,