test(java): run integration test on PR with fine-grained benchmark timers

.github/workflows/java.yaml

@@ -116,7 +116,7 @@ jobs:
         working-directory: google-cloud-java
         run: librarian generate secretmanager
       - name: Run librarian generate all (integration test)
-        if: matrix.task == 'integration' && github.event_name == 'push' && github.ref == 'refs/heads/main'
+        if: matrix.task == 'integration'
         working-directory: google-cloud-java
         run: librarian generate --all
   create-issue-on-failure:

.github/workflows/sidekick.yaml

@@ -28,7 +28,7 @@ jobs:
       - name: Display Go version
         run: go version
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
+        uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
       - name: Display Cargo version
         run: cargo version
       - name: Display rustc version
@@ -49,14 +49,16 @@ jobs:
           persist-credentials: false
       - uses: ./.github/actions/setup-librarian
       - name: Install Rust toolchain
-        uses: dtolnay/rust-toolchain@4cda84d5c5c54efe2404f9d843567869ab1699d4 # stable
-      - uses: ./.github/actions/install-taplo
+        uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c # stable
       - name: Checkout google-cloud-rust
         uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3
         with:
           repository: googleapis/google-cloud-rust
           path: google-cloud-rust
           persist-credentials: false
+      - name: Run librarian install
+        working-directory: google-cloud-rust
+        run: librarian install -v
       - name: Run librarian generate
         working-directory: google-cloud-rust
         run: librarian generate --all

doc/config-schema.md

@@ -642,7 +642,6 @@ This document describes the schema for the librarian.yaml.
 | `api_path` | string | Is the proto path to generate from (e.g., "google/storage/v2"). |
 | `module_type` | string | Is the type of module to generate (e.g., "swift-protobuf", "convert-swift", or empty/"default" for standard GAPIC). |
 | `include_list` | list of string | Is a subset of proto files under the target API path to include. This is typically reserved for special cases to avoid generating unused/dead code. For example, in Storage we need Protobuf gencode for a subset of the protos in the google/type directory. This code is private to the package (google-cloud-storage in Rust, GoogleCloudStorage in Swift). All other files in google/type would be dead code. |
-| `included_ids` | list of string | Is a list of proto IDs to include in generation for this module. If set, only these IDs and their dependencies are generated. |
 | `skipped_ids` | list of string | Is a list of proto IDs to skip in generation for this module. |
 | `module_path` | string | Is the module import path or target containing stubs (used by convert-swift and gRPC transports). |
 
@@ -653,7 +652,6 @@ This document describes the schema for the librarian.yaml.
 | (embedded) | [SwiftDefault](#swiftdefault-configuration) |  |
 | `library_name_override` | string | Overrides the default library name.<br><br>In Swift, each GAPIC package consists of a single product (the library), which contains a single target and module name. For example, the package for the google/cloud/secretmanager/v1 API is called google-cloud-secretmanager-v1, and contains a single product: `GoogleCloudSecretManagerV1`, which in turn contains a single target and module of the same name.<br><br>To use the library applications use this import:<br><br>``` import GoogleCloudSecretManagerV1 ```<br><br>Normally the name is derived from:<br>- If the Protobuf namespace overrides for PHP, Ruby, and C# are consistent, sidekick uses this name.<br>- Otherwise, the name implied by the Protobuf package<br>- Or the package set in the service config yaml file |
 | `include_list` | list of string | Is a subset of proto files under the target API path to include (e.g., ["date.proto", "expr.proto"]). |
-| `included_ids` | list of string | Is a list of proto IDs to include in generation for the package. If set, only these IDs and their dependencies are generated. |
 | `skipped_ids` | list of string | Is a list of proto IDs to skip in generation for the package. |
 | `modules` | list of [SwiftModule](#swiftmodule-configuration) (optional) | Specifies generation targets for veneers and test packages.<br><br>Each module defines a source proto path, and output location. |
 | `package_name_override` | string | Overrides the package name.<br><br>This may be useful if the protobuf package lacks the necessary prefixes, e.g. `grafeas.v1` may be published as `google-grafeas-v1` to match the other packages. |

internal/config/swift.go

@@ -52,10 +52,6 @@ type SwiftPackage struct {
 	// include (e.g., ["date.proto", "expr.proto"]).
 	IncludeList []string `yaml:"include_list,omitempty"`
 
-	// IncludedIDs is a list of proto IDs to include in generation for the package.
-	// If set, only these IDs and their dependencies are generated.
-	IncludedIDs []string `yaml:"included_ids,omitempty"`
-
 	// SkippedIds is a list of proto IDs to skip in generation for the package.
 	SkippedIds []string `yaml:"skipped_ids,omitempty"`
 
@@ -162,10 +158,6 @@ type SwiftModule struct {
 	// in Swift). All other files in google/type would be dead code.
 	IncludeList []string `yaml:"include_list,omitempty"`
 
-	// IncludedIDs is a list of proto IDs to include in generation for this module.
-	// If set, only these IDs and their dependencies are generated.
-	IncludedIDs []string `yaml:"included_ids,omitempty"`
-
 	// SkippedIds is a list of proto IDs to skip in generation for this module.
 	SkippedIds []string `yaml:"skipped_ids,omitempty"`
 

internal/librarian/generate.go

@@ -20,6 +20,7 @@ import (
 	"fmt"
 	"runtime"
 	"strings"
+	"time"
 
 	"github.com/googleapis/librarian/internal/config"
 	"github.com/googleapis/librarian/internal/librarian/dart"
@@ -241,15 +242,29 @@ func generateLibraries(ctx context.Context, cfg *config.Config, libraries []*con
 		}
 		return g.Wait()
 	case config.LanguageJava:
+		genStart := time.Now()
 		for _, library := range libraries {
 			if err := java.Generate(ctx, cfg, library, src); err != nil {
 				return fmt.Errorf("generate library %q (%s): %w", library.Name, cfg.Language, err)
 			}
 		}
+		durGen := time.Since(genStart)
+		fmt.Printf("[BENCHMARK-CI] Phase 1: Java Code Generation Step Completed: %v\n", durGen)
+
+		fmtStart := time.Now()
 		if err := java.Format(ctx, libraries...); err != nil {
 			return fmt.Errorf("format java libraries (%s): %w", cfg.Language, err)
 		}
-		return java.PostGenerate(ctx, ".", cfg)
+		durFmt := time.Since(fmtStart)
+		fmt.Printf("[BENCHMARK-CI] Phase 2: Java Code Formatting Step Completed: %v\n", durFmt)
+
+		postStart := time.Now()
+		if err := java.PostGenerate(ctx, ".", cfg); err != nil {
+			return err
+		}
+		durPost := time.Since(postStart)
+		fmt.Printf("[BENCHMARK-CI] Phase 3: Java Post-Generate Step Completed: %v\n", durPost)
+		return nil
 	case config.LanguageNodejs:
 		g, gctx := errgroup.WithContext(ctx)
 		g.SetLimit(runtime.NumCPU())

internal/librarian/java/format.go

@@ -20,6 +20,7 @@ import (
 	"os"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/googleapis/librarian/internal/command"
 	"github.com/googleapis/librarian/internal/config"
@@ -47,10 +48,12 @@ func Format(ctx context.Context, libraries ...*config.Library) error {
 	for i := 0; i < len(allFiles); i += maxFilesPerFormatBatch {
 		end := min(i+maxFilesPerFormatBatch, len(allFiles))
 		chunk := allFiles[i:end]
+		batchStart := time.Now()
 		args := append([]string{"--replace"}, chunk...)
 		if err := command.RunWithEnv(ctx, env, "google-java-format", args...); err != nil {
 			return fmt.Errorf("failed to format batch [%d:%d]: %w", i, end, err)
 		}
+		fmt.Printf("[BENCHMARK-CI] Format Batch %d files: %v\n", len(chunk), time.Since(batchStart))
 	}
 	return nil
 }

internal/librarian/java/generate.go

@@ -24,6 +24,7 @@ import (
 	"path"
 	"path/filepath"
 	"strings"
+	"time"
 
 	"github.com/googleapis/librarian/internal/config"
 	"github.com/googleapis/librarian/internal/proto"
@@ -164,35 +165,49 @@ func generateAPI(ctx context.Context, params generateAPIParams) error {
 	}
 	// 1. Generate standard Protocol Buffer Java classes.
 	if shouldGenerateProto(javaAPI) {
+		protoStart := time.Now()
 		protoProtos := filterProtos(apiProtos, javaAPI.SkipProtoClassGeneration, primaryDir)
 		protoProtos = append(protoProtos, additionalProtosToGenerateAbs...)
 		args := protoProtocArgs(protoProtos, params.srcCfg, protoDir)
 		if err := runProtoc(ctx, pc, args); err != nil {
 			return fmt.Errorf("failed to generate proto: %w", err)
 		}
+		durProto := time.Since(protoStart)
+		fmt.Printf("[BENCHMARK-CI] API %s Protoc Proto: %v\n", params.api.Path, durProto)
 	}
 	// 2. Generate gRPC service stubs (skipped if transport is rest).
 	transport := params.apiCfg.Transport(config.LanguageJava)
 	if shouldGenerateGRPC(javaAPI) && transport != "rest" {
+		grpcStart := time.Now()
 		if err := runProtoc(ctx, pc, gRPCProtocArgs(apiProtos, params.srcCfg, gRPCDir)); err != nil {
 			return fmt.Errorf("failed to generate gRPC module: %w", err)
 		}
+		durGrpc := time.Since(grpcStart)
+		fmt.Printf("[BENCHMARK-CI] API %s Protoc gRPC: %v\n", params.api.Path, durGrpc)
 	}
 	// 3. Generate GAPIC library.
 	if shouldGenerateGAPIC(javaAPI) || shouldGenerateResourceNames(javaAPI) {
+		optsStart := time.Now()
 		gapicOpts, err := resolveGAPICOptions(params.cfg, params.library, params.api, primaryDir, params.apiCfg)
 		if err != nil {
 			return fmt.Errorf("failed to resolve gapic options: %w", err)
 		}
+		fmt.Printf("[BENCHMARK-CI] API %s GAPIC resolveGAPICOptions: %v\n", params.api.Path, time.Since(optsStart))
+
 		args := gapicProtocArgs(apiProtos, allAdditionalProtosAbs, params.srcCfg, gapicDir, gapicOpts)
+		gapicExecStart := time.Now()
 		if err := runProtoc(ctx, pc, args); err != nil {
 			return fmt.Errorf("failed to generate gapic: %w", err)
 		}
+		fmt.Printf("[BENCHMARK-CI] API %s GAPIC protoc-gen-java_gapic JVM Run: %v\n", params.api.Path, time.Since(gapicExecStart))
 	}
 
+	postStart := time.Now()
 	if err := postProcessAPI(ctx, postParams); err != nil {
 		return fmt.Errorf("failed to post process: %w", err)
 	}
+	durPost := time.Since(postStart)
+	fmt.Printf("[BENCHMARK-CI] API %s postProcessAPI total: %v\n", params.api.Path, durPost)
 	return nil
 }
 

internal/librarian/java/postprocess.go

@@ -114,13 +114,18 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error {
 	// Unzip the temp-codegen.srcjar into temporary {gapicDir} directory.
 	srcjarPath := filepath.Join(gapicDir, "temp-codegen.srcjar")
 	if _, err := os.Stat(srcjarPath); err == nil {
+		unzipStart := time.Now()
 		if err := filesystem.Unzip(ctx, srcjarPath, gapicDir); err != nil {
 			return fmt.Errorf("failed to unzip %s: %w", srcjarPath, err)
 		}
+		fmt.Printf("[BENCHMARK-CI] API %s PostProcess Unzip Srcjar: %v\n", params.apiBase, time.Since(unzipStart))
 	}
+	headerStart := time.Now()
 	if err := addHeaders(params, []string{gRPCDir, protoDir}); err != nil {
 		return err
 	}
+	fmt.Printf("[BENCHMARK-CI] API %s PostProcess Add Headers: %v\n", params.apiBase, time.Since(headerStart))
+
 	if err := copyFiles(params); err != nil {
 		return fmt.Errorf("failed to copy files: %w", err)
 	}
@@ -129,9 +134,11 @@ func postProcessAPI(ctx context.Context, params postProcessParams) error {
 	if params.library != nil {
 		keepSet = toKeepSet(params.library.Keep)
 	}
+	moveStart := time.Now()
 	if err := restructureToLibrary(params, params.outDir, keepSet); err != nil {
 		return fmt.Errorf("failed to restructure to library root: %w", err)
 	}
+	fmt.Printf("[BENCHMARK-CI] API %s PostProcess Restructure Layout: %v\n", params.apiBase, time.Since(moveStart))
 
 	coords := params.coords()
 	// Generate clirr-ignored-differences.xml for the proto module.

internal/librarian/swift/generate.go

@@ -115,11 +115,6 @@ func libraryToModelConfig(library *config.Library, apiCfg *config.API, src *sour
 		specFormat = library.SpecificationFormat
 	}
 
-	var includedIDs []string
-	if library.Swift != nil && len(library.Swift.IncludedIDs) > 0 {
-		includedIDs = library.Swift.IncludedIDs
-	}
-
 	var skippedIDs []string
 	if library.Swift != nil && len(library.Swift.SkippedIds) > 0 {
 		skippedIDs = library.Swift.SkippedIds
@@ -133,8 +128,7 @@ func libraryToModelConfig(library *config.Library, apiCfg *config.API, src *sour
 		Source:              sourceConfig,
 		Protoc:              pc,
 		Override: api.ModelOverride{
-			IncludedIDs: includedIDs,
-			SkippedIDs:  skippedIDs,
+			SkippedIDs: skippedIDs,
 		},
 	}
 	if library.Swift != nil && library.Swift.Discovery != nil {

internal/librarian/swift/generate_module.go

@@ -76,13 +76,6 @@ func moduleToModelConfig(library *config.Library, module *config.SwiftModule, sr
 		sourceConfig.IncludeList = library.Swift.IncludeList
 	}
 
-	var includedIDs []string
-	if len(module.IncludedIDs) > 0 {
-		includedIDs = module.IncludedIDs
-	} else if library.Swift != nil && len(library.Swift.IncludedIDs) > 0 {
-		includedIDs = library.Swift.IncludedIDs
-	}
-
 	var skippedIDs []string
 	if len(module.SkippedIds) > 0 {
 		skippedIDs = module.SkippedIds
@@ -102,8 +95,7 @@ func moduleToModelConfig(library *config.Library, module *config.SwiftModule, sr
 		Source:              sourceConfig,
 		Protoc:              pc,
 		Override: api.ModelOverride{
-			IncludedIDs: includedIDs,
-			SkippedIDs:  skippedIDs,
+			SkippedIDs: skippedIDs,
 		},
 	}
 }

internal/librarian/swift/generate_module_test.go

@@ -363,40 +363,3 @@ func TestModuleToModelConfig_SkippedIds(t *testing.T) {
 		}
 	})
 }
-
-func TestModuleToModelConfig_IncludedIds(t *testing.T) {
-	src := &sources.Sources{}
-
-	t.Run("module level included_ids", func(t *testing.T) {
-		library := &config.Library{
-			Swift: &config.SwiftPackage{
-				IncludedIDs: []string{".google.type.Color"},
-			},
-		}
-		module := &config.SwiftModule{
-			APIPath:     "google/type",
-			IncludedIDs: []string{".google.type.Money"},
-		}
-		modelCfg := moduleToModelConfig(library, module, src)
-		expected := []string{".google.type.Money"}
-		if diff := cmp.Diff(expected, modelCfg.Override.IncludedIDs); diff != "" {
-			t.Errorf("moduleToModelConfig() mismatch (-want +got):\n%s", diff)
-		}
-	})
-
-	t.Run("library level fallback included_ids", func(t *testing.T) {
-		library := &config.Library{
-			Swift: &config.SwiftPackage{
-				IncludedIDs: []string{".google.type.Color"},
-			},
-		}
-		module := &config.SwiftModule{
-			APIPath: "google/type",
-		}
-		modelCfg := moduleToModelConfig(library, module, src)
-		expected := []string{".google.type.Color"}
-		if diff := cmp.Diff(expected, modelCfg.Override.IncludedIDs); diff != "" {
-			t.Errorf("moduleToModelConfig() mismatch (-want +got):\n%s", diff)
-		}
-	})
-}

internal/librarian/swift/generate_test.go

@@ -293,32 +293,6 @@ func TestLibraryToModelConfig(t *testing.T) {
 				},
 			},
 		},
-		{
-			name: "library level included_ids",
-			library: &config.Library{
-				Name:                "google-cloud-secretmanager",
-				Version:             "1.2.3",
-				SpecificationFormat: config.SpecProtobuf,
-				Swift: &config.SwiftPackage{
-					IncludedIDs: []string{".google.cloud.secretmanager.v1.SecretManagerService.GetSecret"},
-				},
-			},
-			api: &config.API{
-				Path: "google/cloud/secretmanager/v1",
-			},
-			want: &parser.ModelConfig{
-				Language:            config.LanguageSwift,
-				SpecificationFormat: config.SpecProtobuf,
-				SpecificationSource: "google/cloud/secretmanager/v1",
-				ServiceConfig:       "google/cloud/secretmanager/v1/secretmanager_v1.yaml",
-				Source: &sources.SourceConfig{
-					ActiveRoots: []string{"googleapis"},
-				},
-				Override: api.ModelOverride{
-					IncludedIDs: []string{".google.cloud.secretmanager.v1.SecretManagerService.GetSecret"},
-				},
-			},
-		},
 		{
 			name: "discovery config",
 			library: &config.Library{

internal/sidekick/api/field.go

@@ -186,6 +186,14 @@ func (f *Field) IsObject() bool {
 	return f.Typez == TypezMessage
 }
 
+// IsWktAny returns true if the field is of type ".google.protobuf.Any"
+//
+// This is a well-known type that requires special treatment in some
+// sidekick gencode <-> Protobuf gencode conversions.
+func (f *Field) IsWktAny() bool {
+	return f.TypezID == WktAnyID
+}
+
 // IsResourceReference returns true if the field is annotated with google.api.resource_reference.
 func (f *Field) IsResourceReference() bool {
 	return f.ResourceReference != nil