Add support for protos in chasm libs (#8182)

## What changed?

Added support for defining protos in chasm libs.

## Why?

Keep everything local to the library.

## How did you test it?
- [x] built
- [x] run locally and tested manually

.gitignore

@@ -24,8 +24,9 @@
 /tctl*
 /tdbg
 
-# proto image
+# proto images
 /proto/image.bin
+/proto/chasm.bin
 # api+google proto dependencies
 /proto/api.binpb
 
@@ -37,3 +38,5 @@
 
 # Git SPR: https://github.com/ejoffe/spr
 .spr.yml
+
+/proto.tmp

Makefile

@@ -100,11 +100,13 @@ endef
 
 PROTO_ROOT := proto
 PROTO_FILES = $(shell find ./$(PROTO_ROOT)/internal -name "*.proto")
+CHASM_PROTO_FILES = $(shell find ./chasm/lib -name "*.proto")
 PROTO_DIRS = $(sort $(dir $(PROTO_FILES)))
 API_BINPB := $(PROTO_ROOT)/api.binpb
 # Note: If you change the value of INTERNAL_BINPB, you'll have to add logic to
 # develop/buf-breaking.sh to handle the old and new values at once.
 INTERNAL_BINPB := $(PROTO_ROOT)/image.bin
+CHASM_BINPB := $(PROTO_ROOT)/chasm.bin
 PROTO_OUT := api
 
 ALL_SRC         := $(shell find . -name "*.go")
@@ -296,11 +298,20 @@ $(INTERNAL_BINPB): $(API_BINPB) $(PROTO_FILES)
 	@printf $(COLOR) "Generate proto image..."
 	@protoc --descriptor_set_in=$(API_BINPB) -I=$(PROTO_ROOT)/internal $(PROTO_FILES) -o $@
 
+$(CHASM_BINPB): $(API_BINPB) $(INTERNAL_BINPB) $(CHASM_PROTO_FILES)
+	@printf $(COLOR) "Generate CHASM proto image..."
+	@protoc --descriptor_set_in=$(API_BINPB):$(INTERNAL_BINPB) -I=. $(CHASM_PROTO_FILES) -o $@
+
 protoc: $(PROTOGEN) $(MOCKGEN) $(GOIMPORTS) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) $(PROTOC_GEN_GO_HELPERS) $(API_BINPB)
-	@env \
+	@go run ./cmd/tools/protogen \
-		PROTOGEN=$(PROTOGEN) MOCKGEN=$(MOCKGEN) GOIMPORTS=$(GOIMPORTS) \
+		-root=$(ROOT) \
-		API_BINPB=$(API_BINPB) PROTO_ROOT=$(PROTO_ROOT) PROTO_OUT=$(PROTO_OUT) \
+		-proto-out=$(PROTO_OUT) \
-		./develop/protoc.sh
+		-proto-root=$(PROTO_ROOT) \
+		-protogen=$(PROTOGEN) \
+		-api-binpb=$(API_BINPB) \
+		-goimports=$(GOIMPORTS) \
+		-mockgen=$(MOCKGEN) \
+		$(PROTO_DIRS)
 
 proto-codegen:
 	@printf $(COLOR) "Generate service clients..."
@@ -368,14 +379,16 @@ lint-api: $(API_LINTER) $(API_BINPB)
 	@printf $(COLOR) "Linting proto API..."
 	$(call silent_exec, $(API_LINTER) --set-exit-status -I=$(PROTO_ROOT)/internal --descriptor-set-in $(API_BINPB) --config=$(PROTO_ROOT)/api-linter.yaml $(PROTO_FILES))
 
-lint-protos: $(BUF) $(INTERNAL_BINPB)
+lint-protos: $(BUF) $(INTERNAL_BINPB) $(CHASM_BINPB)
 	@printf $(COLOR) "Linting proto definitions..."
 	@$(BUF) lint $(INTERNAL_BINPB)
+	@$(BUF) lint --config chasm/lib/buf.yaml $(CHASM_BINPB)
 
 # Edit proto/internal/buf.yaml to exclude specific files from this check.
+# TODO: buf breaking check for CHASM protos.
 buf-breaking: $(BUF) $(API_BINPB) $(INTERNAL_BINPB)
 	@printf $(COLOR) "Run buf breaking proto changes check..."
-	@env BUF=$(BUF) API_BINPB=$(API_BINPB) INTERNAL_BINPB=$(INTERNAL_BINPB) MAIN_BRANCH=$(MAIN_BRANCH) \
+	@env BUF=$(BUF) API_BINPB=$(API_BINPB) INTERNAL_BINPB=$(INTERNAL_BINPB) CHASM_BINPB=$(CHASM_BINPB) MAIN_BRANCH=$(MAIN_BRANCH) \
 		./develop/buf-breaking.sh
 
 shell-check:

api/adminservicemock/v1/service.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package adminservicemock -source api/adminservice/v1/service.pb.go -destination api.new/temporal/server/api/adminservicemock/v1/service.pb.mock.go
+//	mockgen -package adminservicemock -source api/adminservice/v1/service.pb.go -destination api/adminservicemock/v1/service.pb.mock.go
 //
 
 // Package adminservicemock is a generated GoMock package.

api/adminservicemock/v1/service_grpc.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package adminservicemock -source api/adminservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/adminservicemock/v1/service_grpc.pb.mock.go
+//	mockgen -package adminservicemock -source api/adminservice/v1/service_grpc.pb.go -destination api/adminservicemock/v1/service_grpc.pb.mock.go
 //
 
 // Package adminservicemock is a generated GoMock package.

api/historyservicemock/v1/service.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package historyservicemock -source api/historyservice/v1/service.pb.go -destination api.new/temporal/server/api/historyservicemock/v1/service.pb.mock.go
+//	mockgen -package historyservicemock -source api/historyservice/v1/service.pb.go -destination api/historyservicemock/v1/service.pb.mock.go
 //
 
 // Package historyservicemock is a generated GoMock package.

api/historyservicemock/v1/service_grpc.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package historyservicemock -source api/historyservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/historyservicemock/v1/service_grpc.pb.mock.go
+//	mockgen -package historyservicemock -source api/historyservice/v1/service_grpc.pb.go -destination api/historyservicemock/v1/service_grpc.pb.mock.go
 //
 
 // Package historyservicemock is a generated GoMock package.

api/matchingservicemock/v1/service.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package matchingservicemock -source api/matchingservice/v1/service.pb.go -destination api.new/temporal/server/api/matchingservicemock/v1/service.pb.mock.go
+//	mockgen -package matchingservicemock -source api/matchingservice/v1/service.pb.go -destination api/matchingservicemock/v1/service.pb.mock.go
 //
 
 // Package matchingservicemock is a generated GoMock package.

api/matchingservicemock/v1/service_grpc.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package matchingservicemock -source api/matchingservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/matchingservicemock/v1/service_grpc.pb.mock.go
+//	mockgen -package matchingservicemock -source api/matchingservice/v1/service_grpc.pb.go -destination api/matchingservicemock/v1/service_grpc.pb.mock.go
 //
 
 // Package matchingservicemock is a generated GoMock package.

api/testservicemock/v1/service.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package testservicemock -source api/testservice/v1/service.pb.go -destination api.new/temporal/server/api/testservicemock/v1/service.pb.mock.go
+//	mockgen -package testservicemock -source api/testservice/v1/service.pb.go -destination api/testservicemock/v1/service.pb.mock.go
 //
 
 // Package testservicemock is a generated GoMock package.

api/testservicemock/v1/service_grpc.pb.mock.go

@@ -3,7 +3,7 @@
 //
 // Generated by this command:
 //
-//	mockgen -package testservicemock -source api/testservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/testservicemock/v1/service_grpc.pb.mock.go
+//	mockgen -package testservicemock -source api/testservice/v1/service_grpc.pb.go -destination api/testservicemock/v1/service_grpc.pb.mock.go
 //
 
 // Package testservicemock is a generated GoMock package.

chasm/lib/buf.yaml

@@ -0,0 +1,12 @@
+# buf rules for CHASM protos
+version: v1
+deps:
+  - buf.build/googleapis/googleapis
+breaking:
+  use:
+    - WIRE
+lint:
+  use:
+    - DEFAULT
+  except:
+    - PACKAGE_DIRECTORY_MATCH

cmd/tools/protogen/main.go

@@ -0,0 +1,443 @@
+package main
+
+import (
+	"context"
+	"errors"
+	"flag"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"strings"
+
+	"github.com/fatih/color"
+)
+
+var cyan = color.New(color.FgHiCyan, color.Bold)
+
+func info(format string, args ...interface{}) {
+	log.Println(cyan.Sprintf(format, args...))
+}
+
+// runCommand executes a shell command and returns an error if it fails
+func runCommand(ctx context.Context, name string, args ...string) error {
+	cmd := exec.CommandContext(ctx, name, args...)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	return cmd.Run()
+}
+
+func copyRecursive(src, dst string) error {
+	srcInfo, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("error stating source %s: %w", src, err)
+	}
+
+	if !srcInfo.IsDir() {
+		return fmt.Errorf("source %s is not a directory", src)
+	}
+
+	err = os.MkdirAll(dst, srcInfo.Mode())
+	if err != nil {
+		return fmt.Errorf("error creating destination directory %s: %w", dst, err)
+	}
+
+	entries, err := os.ReadDir(src)
+	if err != nil {
+		return fmt.Errorf("error reading source directory %s: %w", src, err)
+	}
+
+	for _, entry := range entries {
+		srcPath := filepath.Join(src, entry.Name())
+		dstPath := filepath.Join(dst, entry.Name())
+		srcInfo, err := entry.Info()
+		if err != nil {
+			return fmt.Errorf("error getting info for %s: %w", srcPath, err)
+		}
+
+		if entry.IsDir() {
+			if err := copyRecursive(srcPath, dstPath); err != nil {
+				return err
+			}
+		} else {
+			// Fix imports in the generated mock file
+			content, err := os.ReadFile(srcPath)
+			if err != nil {
+				return fmt.Errorf("error reading file %s: %w", srcPath, err)
+			}
+
+			if err := os.WriteFile(dstPath, []byte(content), srcInfo.Mode()); err != nil {
+				return fmt.Errorf("error writing file %s: %w", dstPath, err)
+			}
+		}
+	}
+
+	return nil
+}
+
+// exists checks if a path exists
+func exists(path string) bool {
+	_, err := os.Stat(path)
+	return err == nil
+}
+
+func existsDir(path string) bool {
+	stat, err := os.Stat(path)
+	return err == nil && stat.IsDir()
+}
+
+// replaceInFile replaces text in a file using sed-like patterns
+func replaceInFile(filePath, oldPattern, newPattern string) error {
+	content, err := os.ReadFile(filePath)
+	if err != nil {
+		return err
+	}
+
+	// Convert sed pattern to Go regex
+	re, err := regexp.Compile(oldPattern)
+	if err != nil {
+		return err
+	}
+
+	newContent := re.ReplaceAllString(string(content), newPattern)
+	return os.WriteFile(filePath, []byte(newContent), 0644)
+}
+
+type generator struct {
+	rootDir       string
+	tempOut       string
+	tempProtoRoot string
+	protoOut      string
+	protoBackup   string
+	protogen      string
+	apiBinpb      string
+	protoRoot     string
+	goimports     string
+	mockgen       string
+	chasmLibDirs  []string
+}
+
+func newGenerator() (*generator, error) {
+	var gen generator
+	flag.StringVar(&gen.protoOut, "proto-out", "", "Proto output directory (required)")
+	flag.StringVar(&gen.protoRoot, "proto-root", "", "Proto root directory (required)")
+	flag.StringVar(&gen.rootDir, "root", "", "Root directory (required)")
+	flag.StringVar(&gen.protogen, "protogen", "", "Path to protogen binary (required)")
+	flag.StringVar(&gen.apiBinpb, "api-binpb", "", "Path to API binpb file (required)")
+	flag.StringVar(&gen.goimports, "goimports", "", "Path to goimports binary (required)")
+	flag.StringVar(&gen.mockgen, "mockgen", "", "Path to mockgen binary (required)")
+	flag.Parse()
+
+	// Validate required flags
+	if gen.protoOut == "" || gen.protoRoot == "" || gen.rootDir == "" || gen.protogen == "" ||
+		gen.apiBinpb == "" || gen.goimports == "" || gen.mockgen == "" {
+		flag.Usage()
+		return nil, errors.New("all flags are required")
+	}
+
+	chasmDir := filepath.Join(gen.rootDir, "chasm", "lib")
+
+	ls, err := os.ReadDir(chasmDir)
+	if err != nil {
+		return nil, fmt.Errorf("error reading chasm/lib directory %s: %w", chasmDir, err)
+	}
+	gen.chasmLibDirs = make([]string, 0, len(ls))
+	for _, entry := range ls {
+		if !entry.IsDir() {
+			continue
+		}
+		gen.chasmLibDirs = append(gen.chasmLibDirs, filepath.Join(chasmDir, entry.Name()))
+	}
+	gen.tempOut = gen.protoOut + ".new"
+	gen.tempProtoRoot = gen.protoRoot + ".tmp"
+	gen.protoBackup = gen.protoOut + ".old"
+	return &gen, nil
+}
+
+func (g *generator) removeExistingGenDirs() error {
+	for _, dir := range g.chasmLibDirs {
+		genDir := filepath.Join(dir, "gen")
+		if err := os.RemoveAll(genDir); err != nil {
+			return fmt.Errorf("error removing directory %s: %w", genDir, err)
+		}
+	}
+	return nil
+}
+
+func (g *generator) backupProtos() error {
+	if exists(g.protoOut) {
+		if err := os.Rename(g.protoOut, g.protoBackup); err != nil {
+			return fmt.Errorf("error backing up proto output directory %s to %s: %w", g.protoOut, g.protoBackup, err)
+		}
+	}
+	return nil
+}
+
+func (g *generator) prepareTempDirs() error {
+	// Remove and create new directory
+	if err := os.RemoveAll(g.tempOut); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error removing directory %s: %w", g.tempOut, err)
+	}
+	if err := os.MkdirAll(g.tempOut, 0755); err != nil {
+		return fmt.Errorf("error creating directory %s: %w", g.tempOut, err)
+	}
+	if err := os.RemoveAll(g.tempProtoRoot); err != nil && !os.IsNotExist(err) {
+		return fmt.Errorf("error removing directory %s: %w", g.tempOut, err)
+	}
+	if err := copyRecursive(g.protoRoot, g.tempProtoRoot); err != nil {
+		return fmt.Errorf("error copying proto root %s to %s: %w", g.protoRoot, g.tempProtoRoot, err)
+	}
+	return nil
+}
+
+func (g *generator) copyChasmLibProtos() error {
+	for _, dir := range g.chasmLibDirs {
+		protoDir := filepath.Join(dir, "proto")
+		if !existsDir(protoDir) {
+			continue
+		}
+		destDir := filepath.Join(g.tempProtoRoot, "internal", "temporal", "server", "chasm", "lib", filepath.Base(dir))
+		if err := os.MkdirAll(destDir, 0755); err != nil {
+			return fmt.Errorf("error creating directory %s: %w", destDir, err)
+		}
+		if err := copyRecursive(protoDir, filepath.Join(destDir, "proto")); err != nil {
+			return fmt.Errorf("error copying proto files from %s to %s: %w", protoDir, destDir, err)
+		}
+		protos, err := filepath.Glob(filepath.Join(destDir, "proto", "**", "*.proto"))
+		if err != nil {
+			return fmt.Errorf("error finding proto files in %s: %w", destDir, err)
+		}
+		for _, proto := range protos {
+			if err := replaceInFile(proto, `import "chasm`, `import "temporal/server/chasm`); err != nil {
+				return fmt.Errorf("error updating import path in %s: %w", proto, err)
+			}
+		}
+	}
+	return nil
+}
+
+func (g *generator) runProtogen(ctx context.Context) error {
+	// Run protogen
+	protoArgs := []string{
+		"--descriptor_set_in=" + g.apiBinpb,
+		"--root=" + filepath.Join(g.tempProtoRoot, "internal"),
+		"--rewrite-enum=BuildId_State:BuildId",
+		"--output=" + g.tempOut,
+		"-p", "go-grpc_out=paths=source_relative:" + g.tempOut,
+		"-p", "go-helpers_out=paths=source_relative:" + g.tempOut,
+	}
+
+	if err := runCommand(ctx, g.protogen, protoArgs...); err != nil {
+		return fmt.Errorf("error running protogen: %w", err)
+	}
+	return nil
+}
+
+func (g *generator) runGoImports(ctx context.Context) error {
+	if err := runCommand(ctx, g.goimports, "-w", g.tempOut); err != nil {
+		return fmt.Errorf("error running goimports: %w", err)
+	}
+
+	return nil
+}
+
+func (g *generator) generateProtoMocks(ctx context.Context) error {
+	// Find service.pb.go and service_grpc.pb.go files
+	return filepath.Walk(g.tempOut, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		if strings.HasSuffix(path, "service.pb.go") || strings.HasSuffix(path, "service_grpc.pb.go") {
+			// Convert service/ to servicemock/ and .go to .mock.go
+			dst := strings.ReplaceAll(path, "service/", "servicemock/")
+			dst = strings.ReplaceAll(dst, ".go", ".mock.go")
+
+			// Get package name
+			dstDir := filepath.Dir(dst)
+			pkg := filepath.Base(filepath.Dir(dstDir))
+
+			// Create destination directory
+			if err := os.MkdirAll(dstDir, 0755); err != nil {
+				return fmt.Errorf("error creating directory %s: %v", dstDir, err)
+			}
+
+			// Run mockgen
+			mockgenArgs := []string{
+				"-package", pkg,
+				"-source", path,
+				"-destination", dst,
+			}
+
+			if err := runCommand(ctx, g.mockgen, mockgenArgs...); err != nil {
+				return fmt.Errorf("error running mockgen: %v", err)
+			}
+
+			// Fix imports in the generated mock file
+			content, err := os.ReadFile(dst)
+			if err != nil {
+				return fmt.Errorf("error reading file %s: %v", dst, err)
+			}
+
+			// Replace the incorrect import path
+			newContent := strings.ReplaceAll(string(content), g.tempOut+"/temporal/server/", "")
+
+			if err := os.WriteFile(dst, []byte(newContent), 0644); err != nil {
+				return fmt.Errorf("error writing file %s: %v", dst, err)
+			}
+		}
+		return nil
+	})
+}
+
+func (g *generator) modifyHistoryServiceFile() error {
+	historyServiceFile := filepath.Join(g.tempOut, "temporal", "server", "api", "historyservice", "v1", "service_grpc.pb.go")
+
+	// Replace GetWorkflowExecutionHistory method signature
+	err := replaceInFile(historyServiceFile,
+		`GetWorkflowExecutionHistory\(context\.Context, \*GetWorkflowExecutionHistoryRequest\) \(\*GetWorkflowExecutionHistoryResponse, error\)`,
+		`GetWorkflowExecutionHistory(context.Context, *GetWorkflowExecutionHistoryRequest) (*GetWorkflowExecutionHistoryResponseWithRaw, error)`)
+	if err != nil {
+		return fmt.Errorf("error modifying GetWorkflowExecutionHistory: %w", err)
+	}
+
+	// Replace RecordWorkflowTaskStarted method signature
+	err = replaceInFile(historyServiceFile,
+		`RecordWorkflowTaskStarted\(context\.Context, \*RecordWorkflowTaskStartedRequest\) \(\*RecordWorkflowTaskStartedResponse, error\)`,
+		`RecordWorkflowTaskStarted(context.Context, *RecordWorkflowTaskStartedRequest) (*RecordWorkflowTaskStartedResponseWithRawHistory, error)`)
+	if err != nil {
+		return fmt.Errorf("error modifying RecordWorkflowTaskStarted: %w", err)
+	}
+
+	return nil
+}
+
+func (g *generator) moveProtoFiles() error {
+	sourceApiDir := filepath.Join(g.tempOut, "temporal", "server", "api")
+	return os.Rename(sourceApiDir, g.protoOut)
+}
+
+func (g *generator) moveGeneratedChasmFiles() error {
+	sourceChasmDir := filepath.Join(g.tempOut, "temporal", "server", "chasm")
+
+	return filepath.Walk(sourceChasmDir, func(path string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Calculate relative path from sourceChasmDir
+		relPath, err := filepath.Rel(sourceChasmDir, path)
+		if err != nil {
+			return err
+		}
+		// Transform relPath to match the destination structure
+		// e.g., "lib/<name>/proto/v1/*" -> "lib/<name>/gen/<name>pb/*"
+		re := regexp.MustCompile(`^lib/([^/]+)/proto/(v\d+)/(.*)$`)
+		parts := re.FindStringSubmatch(relPath)
+		if len(parts) < 4 {
+			return nil
+		}
+		dstPath := filepath.Join("chasm", "lib", parts[1], "gen", parts[1]+"pb", parts[2], parts[3])
+		dirName := filepath.Dir(dstPath) // Ensure the destination path is a base path
+
+		if err := os.MkdirAll(dirName, 0755); err != nil {
+			return fmt.Errorf("error creating directory %s: %w", dirName, err)
+		}
+
+		if err := os.Rename(path, dstPath); err != nil {
+			return fmt.Errorf("error moving file %s to %s: %w", path, dstPath, err)
+		}
+		return nil
+	})
+}
+
+func (g *generator) cleanup(restoreOld bool) error {
+	// Remove temporary directories
+	if err := os.RemoveAll(g.tempProtoRoot); err != nil {
+		return fmt.Errorf("error removing temporary proto root %s: %w", g.tempProtoRoot, err)
+	}
+	if err := os.RemoveAll(g.tempOut); err != nil {
+		return fmt.Errorf("error removing temporary output directory %s: %w", g.tempOut, err)
+	}
+	if restoreOld {
+		// Restore the old proto output directory
+		if exists(g.protoBackup) {
+			if err := os.Rename(g.protoBackup, g.protoOut); err != nil {
+				return fmt.Errorf("error restoring proto output directory from %s to %s: %w", g.protoBackup, g.protoOut, err)
+			}
+		}
+	} else {
+		// Remove the old proto output directory if it exists
+		if err := os.RemoveAll(g.protoBackup); err != nil && !os.IsNotExist(err) {
+			return fmt.Errorf("error removing backup proto output directory %s: %w", g.protoBackup, err)
+		}
+	}
+
+	return nil
+}
+
+func main() {
+	ctx := context.Background()
+	gen, err := newGenerator()
+	if err != nil {
+		log.Fatalln("Error creating generator:", err)
+	}
+
+	info("Removing existing CHASM gen directories...")
+	if err := gen.removeExistingGenDirs(); err != nil {
+		log.Fatalln("Error removing existing CHASM gen directories:", err)
+	}
+
+	info("Backing up existing proto output directory...")
+	if err := gen.backupProtos(); err != nil {
+		log.Fatalln("Error backing up proto output directory:", err)
+	}
+
+	genErr := generate(ctx, gen)
+	cleanupErr := gen.cleanup(genErr != nil)
+	if cleanupErr != nil {
+		cleanupErr = fmt.Errorf("error cleaning up after generation: %w", cleanupErr)
+	}
+	err = errors.Join(genErr, cleanupErr)
+	if err != nil {
+		log.Fatalln("Generation failed:", err)
+	}
+}
+
+func generate(ctx context.Context, gen *generator) error {
+	info("Preparing temp directories...")
+	if err := gen.prepareTempDirs(); err != nil {
+		return fmt.Errorf("error preparing new gen directory: %w", err)
+	}
+	info("Copying CHASM lib protos...")
+	if err := gen.copyChasmLibProtos(); err != nil {
+		return fmt.Errorf("error copying CHASM lib protos: %w", err)
+	}
+	info("Running protoc for proto files...")
+	if err := gen.runProtogen(ctx); err != nil {
+		return fmt.Errorf("error running protogen: %w", err)
+	}
+	info("Running goimports for proto files...")
+	if err := gen.runGoImports(ctx); err != nil {
+		return fmt.Errorf("error running goimports: %w", err)
+	}
+	info("Generating proto mocks...")
+	if err := gen.generateProtoMocks(ctx); err != nil {
+		return fmt.Errorf("error generating mock files: %w", err)
+	}
+	info("Modifying history service server interface...")
+	if err := gen.modifyHistoryServiceFile(); err != nil {
+		return fmt.Errorf("error modifying history service file: %w", err)
+	}
+	info("Moving proto files into place...")
+	if err := gen.moveProtoFiles(); err != nil {
+		return fmt.Errorf("error moving proto files: %w", err)
+	}
+	info("Moving generated CHASM files into place...")
+	if err := gen.moveGeneratedChasmFiles(); err != nil {
+		return fmt.Errorf("error moving CHASM files: %w", err)
+	}
+	return nil
+}

develop/buf-breaking.sh

@@ -23,6 +23,7 @@ set -eu -o pipefail
 : "${BUF:=buf}"
 : "${API_BINPB:=proto/api.binpb}"
 : "${INTERNAL_BINPB:=proto/image.bin}"
+: "${CHASM_BINPB:=proto/chasm.bin}"
 : "${MAIN_BRANCH:=main}"
 
 : "${COMMIT:=$(git rev-parse HEAD)}"
@@ -45,6 +46,7 @@ fi
 # If invoked from the Makefile, this should already be done. This is just in
 # case this is being run manually.
 $MAKE "$INTERNAL_BINPB"
+$MAKE "$CHASM_BINPB"
 
 tmp=$(mktemp --tmpdir -d temporal-buf-breaking.XXXXXXXXX)
 trap 'rm -rf $tmp' EXIT
@@ -60,7 +62,14 @@ check_against_commit() {
     $MAKE -C "$tmp" "$INTERNAL_BINPB"
     $BUF breaking "$INTERNAL_BINPB" --against "$tmp/$INTERNAL_BINPB" --config proto/internal/buf.yaml
   else
-    yellow "$name commit is too old to support breaking check"
+    yellow "$name commit is too old to support breaking check for internal protos"
+  fi
+
+  if grep -q CHASM_BINPB "$tmp/Makefile"; then
+    $MAKE -C "$tmp" "$CHASM_BINPB"
+    $BUF breaking "$CHASM_BINPB" --against "$tmp/$CHASM_BINPB" --config chasm/lib/buf.yaml
+  else
+    yellow "$name commit is too old to support breaking check for chasm protos"
   fi
 }
 

develop/protoc.sh

@@ -1,49 +0,0 @@
-#!/usr/bin/env bash
-
-set -eu -o pipefail
-
-color() { printf "\e[1;36m%s\e[0m\n" "$*"; }
-
-api=$PROTO_OUT
-new=$api.new
-
-color "Generating protos..."
-
-rm -rf "$new" && mkdir -p "$new"
-
-$PROTOGEN \
-  --descriptor_set_in="$API_BINPB" \
-  --root="$PROTO_ROOT"/internal \
-  --rewrite-enum=BuildId_State:BuildId \
-  --output="$new" \
-  -p go-grpc_out=paths=source_relative:"$new" \
-  -p go-helpers_out=paths=source_relative:"$new"
-
-color "Run goimports for proto files..."
-$GOIMPORTS -w "$new"
-
-color "Generate proto mocks..."
-find "$new" -name service.pb.go -o -name service_grpc.pb.go | while read -r src; do
-  dst=$(echo "$src" | sed -e 's,service/,servicemock/,' -e 's,[.]go$,.mock.go,')
-  pkg=$(basename "$(dirname "$(dirname "$dst")")")
-  $MOCKGEN -package "$pkg" -source "$src" -destination "$dst"
-  # Since we're generating mocks from files in incorrect locations, mockgen
-  # writes incorrect imports. Fix them manually. The awkward sed/rm invocation
-  # is to work on linux and macos.
-  sed -i.replaced "s,$new/temporal/server/,," "$dst"
-  rm -f "$dst.replaced"
-done
-
-color "Modify history service server interface..."
-sed -i.bak \
-    -e 's/GetWorkflowExecutionHistory(context\.Context, \*GetWorkflowExecutionHistoryRequest) (\*GetWorkflowExecutionHistoryResponse, error)/GetWorkflowExecutionHistory(context.Context, *GetWorkflowExecutionHistoryRequest) (*GetWorkflowExecutionHistoryResponseWithRaw, error)/g' \
-    -e 's/RecordWorkflowTaskStarted(context\.Context, \*RecordWorkflowTaskStartedRequest) (\*RecordWorkflowTaskStartedResponse, error)/RecordWorkflowTaskStarted(context.Context, *RecordWorkflowTaskStartedRequest) (*RecordWorkflowTaskStartedResponseWithRawHistory, error)/g' \
-    "$new"/temporal/server/api/historyservice/v1/service_grpc.pb.go
-rm "$new"/temporal/server/api/historyservice/v1/service_grpc.pb.go.bak
-
-color "Moving proto files into place..."
-old=$api.old
-[[ -d "$api" ]] && mv -f "$api" "$old"
-mkdir -p "$api"
-mv -f "$new"/temporal/server/api/* "$api"/
-rm -rf "$new" "$old"