Add support for LSIF Typed (#732)

* Add support for LSIF Typed

Running `src lsif upload` now converts LSIF Typed into LSIF Graph in the
following scenarios:

- When the provided `-file` flag matches the file extension ".lsif-typed"
- When the provided `-file` flag points to a non-existent file, while
  the sibling `*.lsif-typed` file exists.

Fixes https://github.com/sourcegraph/sourcegraph/issues/27405

* Fix ordering of how we initialize `*output.Output`

* Add test cases for `handleLSIFTyped` function.

* Remove redundant blank line

* Use `t.TempDir()` to create temporary directory

Author: olafurpg

cmd/src/lsif_upload.go

@@ -57,8 +57,7 @@ Examples:
 func handleLSIFUpload(args []string) error {
 	ctx := context.Background()
 
-	err := parseAndValidateLSIFUploadFlags(args)
-	out := lsifUploadOutput()
+	out, err := parseAndValidateLSIFUploadFlags(args)
 	if !lsifUploadFlags.json {
 		if out != nil {
 			printInferredArguments(out)
@@ -119,22 +118,6 @@ func handleLSIFUpload(args []string) error {
 	return nil
 }
 
-// lsifUploadOutput returns an output object that should be used to print the progres
-// of requests made during this upload. If -json, -no-progress, or -trace>0 is given,
-// then no output object is defined.
-//
-// For -no-progress and -trace>0 conditions, emergency loggers will be used to display
-// inferred arguments and the URL at which processing status is shown.
-func lsifUploadOutput() (out *output.Output) {
-	if lsifUploadFlags.json || lsifUploadFlags.noProgress || lsifUploadFlags.verbosity > 0 {
-		return nil
-	}
-
-	return output.NewOutput(flag.CommandLine.Output(), output.OutputOpts{
-		Verbose: true,
-	})
-}
-
 // lsifUploadOptions creates a set of upload options given the values in the flags.
 func lsifUploadOptions(out *output.Output) upload.UploadOptions {
 	var associatedIndexID *int

cmd/src/lsif_upload_flags.go

@@ -7,7 +7,11 @@ import (
 	"strings"
 
 	"github.com/sourcegraph/sourcegraph/lib/errors"
+	"github.com/sourcegraph/sourcegraph/lib/output"
+	"google.golang.org/protobuf/proto"
 
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsif/protocol/reader"
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsiftyped"
 	"github.com/sourcegraph/sourcegraph/lib/codeintel/upload"
 
 	"github.com/sourcegraph/src-cli/internal/api"
@@ -84,11 +88,13 @@ func init() {
 //
 // On success, the global lsifUploadFlags object will be populated with valid values. An
 // error is returned on failure.
-func parseAndValidateLSIFUploadFlags(args []string) error {
+func parseAndValidateLSIFUploadFlags(args []string) (*output.Output, error) {
 	if err := lsifUploadFlagSet.Parse(args); err != nil {
-		return err
+		return nil, err
 	}
 
+	out := lsifUploadOutput()
+
 	// extract only the -insecure-skip-verify flag so we dont get 'flag provided but not defined'
 	var insecureSkipVerifyFlag []string
 	for _, s := range args {
@@ -102,11 +108,15 @@ func parseAndValidateLSIFUploadFlags(args []string) error {
 	// and maybe we'll use some in the future
 	lsifUploadFlags.apiFlags = api.NewFlags(apiClientFlagSet)
 	if err := apiClientFlagSet.Parse(insecureSkipVerifyFlag); err != nil {
-		return err
+		return nil, err
+	}
+
+	if err := handleLSIFTyped(out); err != nil {
+		return nil, err
 	}
 
 	if inferenceErrors := inferMissingLSIFUploadFlags(); len(inferenceErrors) > 0 {
-		return errorWithHint{
+		return nil, errorWithHint{
 			err: inferenceErrors[0].err, hint: strings.Join([]string{
 				fmt.Sprintf(
 					"Unable to determine %s from environment. Check your working directory or supply -%s={value} explicitly",
@@ -118,17 +128,93 @@ func parseAndValidateLSIFUploadFlags(args []string) error {
 	}
 
 	if err := validateLSIFUploadFlags(); err != nil {
-		return err
+		return nil, err
 	}
 
-	return nil
+	return out, nil
+}
+
+// lsifUploadOutput returns an output object that should be used to print the progres
+// of requests made during this upload. If -json, -no-progress, or -trace>0 is given,
+// then no output object is defined.
+//
+// For -no-progress and -trace>0 conditions, emergency loggers will be used to display
+// inferred arguments and the URL at which processing status is shown.
+func lsifUploadOutput() (out *output.Output) {
+	if lsifUploadFlags.json || lsifUploadFlags.noProgress || lsifUploadFlags.verbosity > 0 {
+		return nil
+	}
+
+	return output.NewOutput(flag.CommandLine.Output(), output.OutputOpts{
+		Verbose: true,
+	})
 }
 
 type argumentInferenceError struct {
 	argument string
 	err      error
 }
 
+func handleLSIFTyped(out *output.Output) error {
+	if strings.HasSuffix(lsifUploadFlags.file, ".lsif-typed") {
+		// The user explicitly passed in a -file flag that points to an LSIF Typed index.
+		inputFile := lsifUploadFlags.file
+		outputFile := strings.TrimSuffix(inputFile, "-typed")
+		lsifUploadFlags.file = outputFile
+		return convertLSIFTypedToLSIFGraph(out, inputFile, outputFile)
+	}
+
+	if _, err := os.Stat(lsifUploadFlags.file); err == nil {
+		// Do nothing, the provided -flag flag points to an existing
+		// file that does not have the file extension `*.lsif-typed`.
+		return nil
+	}
+
+	lsifTypedFile := lsifUploadFlags.file + "-typed"
+	if _, err := os.Stat(lsifTypedFile); os.IsNotExist(err) {
+		// The inferred path of the sibling `*.lsif-typed` file does not exist.
+		return nil
+	}
+
+	// The provided -file flag points to an `*.lsif` file that doesn't exist
+	// so we convert the sibling `*.lsif-typed` file (which we confirmed exists).
+	return convertLSIFTypedToLSIFGraph(out, lsifTypedFile, lsifUploadFlags.file)
+}
+
+// Reads the LSIF Typed encoded input file and writes the corresponding LSIF
+// Graph encoded output file.
+func convertLSIFTypedToLSIFGraph(out *output.Output, inputFile, outputFile string) error {
+	out.Writef("%s  Converting %s into %s", output.EmojiInfo, inputFile, outputFile)
+	tmp, err := os.Create(outputFile)
+	if err != nil {
+		return err
+	}
+	defer tmp.Close()
+
+	data, err := os.ReadFile(inputFile)
+	if err != nil {
+		panic(err)
+	}
+	index := lsiftyped.Index{}
+	err = proto.Unmarshal(data, &index)
+	if err != nil {
+		panic(errors.Wrapf(err, "failed to parse protobuf file '%s'", inputFile))
+	}
+	els, err := reader.ConvertTypedIndexToGraphIndex(&index)
+	if err != nil {
+		panic(errors.Wrapf(err, "failed reader.ConvertTypedIndexToGraphIndex"))
+	}
+	err = reader.WriteNDJSON(reader.ElementsToJsonElements(els), tmp)
+	if err != nil {
+		panic(err)
+	}
+	err = tmp.Close()
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
 // inferMissingLSIFUploadFlags updates the flags values which were not explicitly
 // supplied by the user with default values inferred from the current git state and
 // filesystem.

cmd/src/lsif_upload_flags_test.go

@@ -0,0 +1,77 @@
+package main
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/sourcegraph/sourcegraph/lib/codeintel/lsiftyped"
+	"google.golang.org/protobuf/proto"
+)
+
+var exampleLsifTypedIndex = lsiftyped.Index{
+	Metadata: &lsiftyped.Metadata{
+		TextDocumentEncoding: lsiftyped.TextEncoding_UTF8,
+		ToolInfo: &lsiftyped.ToolInfo{
+			Name:    "hello",
+			Version: "1.0.0",
+		},
+	},
+}
+
+var exampleLsifGraphString = `{"id":1,"version":"0.4.3","positionEncoding":"utf-8","toolInfo":{"name":"hello","version":"1.0.0"},"type":"vertex","label":"metaData"}
+`
+
+func exampleLsifTypedBytes(t *testing.T) []byte {
+	bytes, err := proto.Marshal(&exampleLsifTypedIndex)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return bytes
+}
+
+func createTempLsifTypedFile(t *testing.T) (typedFile, graphFile string) {
+	dir := t.TempDir()
+	typedFile = filepath.Join(dir, "dump.lsif-typed")
+	graphFile = filepath.Join(dir, "dump.lsif")
+	err := os.WriteFile(typedFile, exampleLsifTypedBytes(t), 0755)
+	if err != nil {
+		t.Fatal(err)
+	}
+	return typedFile, graphFile
+}
+
+func assertLsifGraphOutput(t *testing.T, lsifGraphFile, expectedGraphString string) {
+	out := lsifUploadOutput()
+	handleLSIFTyped(out)
+	lsifGraph, err := os.ReadFile(lsifGraphFile)
+	if err != nil {
+		t.Fatal(err)
+	}
+	obtained := string(lsifGraph)
+	if obtained != expectedGraphString {
+		t.Fatalf("unexpected LSIF output %s", obtained)
+	}
+	if lsifGraphFile != lsifUploadFlags.file {
+		t.Fatalf("unexpected lsifUploadFlag.file value %s, expected %s", lsifUploadFlags.file, lsifGraphFile)
+	}
+}
+
+func TestImplicitlyConvertLsifTypedIntoGraph(t *testing.T) {
+	_, graphFile := createTempLsifTypedFile(t)
+	lsifUploadFlags.file = graphFile
+	assertLsifGraphOutput(t, graphFile, exampleLsifGraphString)
+}
+
+func TestImplicitlyIgnoreLsifTyped(t *testing.T) {
+	_, graphFile := createTempLsifTypedFile(t)
+	lsifUploadFlags.file = graphFile
+	os.WriteFile(graphFile, []byte("hello world"), 0755)
+	assertLsifGraphOutput(t, graphFile, "hello world")
+}
+
+func TestExplicitlyConvertLsifTypedIntoGraph(t *testing.T) {
+	typedFile, graphFile := createTempLsifTypedFile(t)
+	lsifUploadFlags.file = typedFile
+	assertLsifGraphOutput(t, graphFile, exampleLsifGraphString)
+}

go.mod

@@ -22,6 +22,7 @@ require (
 	github.com/sourcegraph/sourcegraph/lib v0.0.0-20220414150621-eeb00fcedd88
 	github.com/stretchr/testify v1.7.1
 	golang.org/x/net v0.0.0-20220325170049-de3da57026de
+	google.golang.org/protobuf v1.27.1
 	gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b
 	jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7
 )

go.sum

@@ -491,6 +491,7 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=