From 4c59cd15d7b68e3f583555e19b2e75f5cd957130 Mon Sep 17 00:00:00 2001 From: Roey Berman Date: Fri, 15 Aug 2025 12:09:58 -0600 Subject: [PATCH] Add support for protos in chasm libs (#8182) ## What changed? Added support for defining protos in chasm libs. ## Why? Keep everything local to the library. ## How did you test it? - [x] built - [x] run locally and tested manually --- .gitignore | 5 +- Makefile | 25 +- api/adminservicemock/v1/service.pb.mock.go | 2 +- .../v1/service_grpc.pb.mock.go | 2 +- api/historyservicemock/v1/service.pb.mock.go | 2 +- .../v1/service_grpc.pb.mock.go | 2 +- api/matchingservicemock/v1/service.pb.mock.go | 2 +- .../v1/service_grpc.pb.mock.go | 2 +- api/testservicemock/v1/service.pb.mock.go | 2 +- .../v1/service_grpc.pb.mock.go | 2 +- chasm/lib/buf.yaml | 12 + cmd/tools/protogen/main.go | 443 ++++++++++++++++++ develop/buf-breaking.sh | 11 +- develop/protoc.sh | 49 -- 14 files changed, 496 insertions(+), 65 deletions(-) create mode 100644 chasm/lib/buf.yaml create mode 100644 cmd/tools/protogen/main.go delete mode 100755 develop/protoc.sh diff --git a/.gitignore b/.gitignore index e8778dd55e..34872c91c7 100644 --- a/.gitignore +++ b/.gitignore @@ -24,8 +24,9 @@ /tctl* /tdbg -# proto image +# proto images /proto/image.bin +/proto/chasm.bin # api+google proto dependencies /proto/api.binpb @@ -37,3 +38,5 @@ # Git SPR: https://github.com/ejoffe/spr .spr.yml + +/proto.tmp diff --git a/Makefile b/Makefile index 2c434d38de..3f0ccb8c97 100644 --- a/Makefile +++ b/Makefile @@ -100,11 +100,13 @@ endef PROTO_ROOT := proto PROTO_FILES = $(shell find ./$(PROTO_ROOT)/internal -name "*.proto") +CHASM_PROTO_FILES = $(shell find ./chasm/lib -name "*.proto") PROTO_DIRS = $(sort $(dir $(PROTO_FILES))) API_BINPB := $(PROTO_ROOT)/api.binpb # Note: If you change the value of INTERNAL_BINPB, you'll have to add logic to # develop/buf-breaking.sh to handle the old and new values at once. INTERNAL_BINPB := $(PROTO_ROOT)/image.bin +CHASM_BINPB := $(PROTO_ROOT)/chasm.bin PROTO_OUT := api ALL_SRC := $(shell find . -name "*.go") @@ -296,11 +298,20 @@ $(INTERNAL_BINPB): $(API_BINPB) $(PROTO_FILES) @printf $(COLOR) "Generate proto image..." @protoc --descriptor_set_in=$(API_BINPB) -I=$(PROTO_ROOT)/internal $(PROTO_FILES) -o $@ +$(CHASM_BINPB): $(API_BINPB) $(INTERNAL_BINPB) $(CHASM_PROTO_FILES) + @printf $(COLOR) "Generate CHASM proto image..." + @protoc --descriptor_set_in=$(API_BINPB):$(INTERNAL_BINPB) -I=. $(CHASM_PROTO_FILES) -o $@ + protoc: $(PROTOGEN) $(MOCKGEN) $(GOIMPORTS) $(PROTOC_GEN_GO) $(PROTOC_GEN_GO_GRPC) $(PROTOC_GEN_GO_HELPERS) $(API_BINPB) - @env \ - PROTOGEN=$(PROTOGEN) MOCKGEN=$(MOCKGEN) GOIMPORTS=$(GOIMPORTS) \ - API_BINPB=$(API_BINPB) PROTO_ROOT=$(PROTO_ROOT) PROTO_OUT=$(PROTO_OUT) \ - ./develop/protoc.sh + @go run ./cmd/tools/protogen \ + -root=$(ROOT) \ + -proto-out=$(PROTO_OUT) \ + -proto-root=$(PROTO_ROOT) \ + -protogen=$(PROTOGEN) \ + -api-binpb=$(API_BINPB) \ + -goimports=$(GOIMPORTS) \ + -mockgen=$(MOCKGEN) \ + $(PROTO_DIRS) proto-codegen: @printf $(COLOR) "Generate service clients..." @@ -368,14 +379,16 @@ lint-api: $(API_LINTER) $(API_BINPB) @printf $(COLOR) "Linting proto API..." $(call silent_exec, $(API_LINTER) --set-exit-status -I=$(PROTO_ROOT)/internal --descriptor-set-in $(API_BINPB) --config=$(PROTO_ROOT)/api-linter.yaml $(PROTO_FILES)) -lint-protos: $(BUF) $(INTERNAL_BINPB) +lint-protos: $(BUF) $(INTERNAL_BINPB) $(CHASM_BINPB) @printf $(COLOR) "Linting proto definitions..." @$(BUF) lint $(INTERNAL_BINPB) + @$(BUF) lint --config chasm/lib/buf.yaml $(CHASM_BINPB) # Edit proto/internal/buf.yaml to exclude specific files from this check. +# TODO: buf breaking check for CHASM protos. buf-breaking: $(BUF) $(API_BINPB) $(INTERNAL_BINPB) @printf $(COLOR) "Run buf breaking proto changes check..." - @env BUF=$(BUF) API_BINPB=$(API_BINPB) INTERNAL_BINPB=$(INTERNAL_BINPB) MAIN_BRANCH=$(MAIN_BRANCH) \ + @env BUF=$(BUF) API_BINPB=$(API_BINPB) INTERNAL_BINPB=$(INTERNAL_BINPB) CHASM_BINPB=$(CHASM_BINPB) MAIN_BRANCH=$(MAIN_BRANCH) \ ./develop/buf-breaking.sh shell-check: diff --git a/api/adminservicemock/v1/service.pb.mock.go b/api/adminservicemock/v1/service.pb.mock.go index e05caae5a9..250d7c21b7 100644 --- a/api/adminservicemock/v1/service.pb.mock.go +++ b/api/adminservicemock/v1/service.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package adminservicemock -source api/adminservice/v1/service.pb.go -destination api.new/temporal/server/api/adminservicemock/v1/service.pb.mock.go +// mockgen -package adminservicemock -source api/adminservice/v1/service.pb.go -destination api/adminservicemock/v1/service.pb.mock.go // // Package adminservicemock is a generated GoMock package. diff --git a/api/adminservicemock/v1/service_grpc.pb.mock.go b/api/adminservicemock/v1/service_grpc.pb.mock.go index 36ae09b2a3..53f4338dbd 100644 --- a/api/adminservicemock/v1/service_grpc.pb.mock.go +++ b/api/adminservicemock/v1/service_grpc.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package adminservicemock -source api/adminservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/adminservicemock/v1/service_grpc.pb.mock.go +// mockgen -package adminservicemock -source api/adminservice/v1/service_grpc.pb.go -destination api/adminservicemock/v1/service_grpc.pb.mock.go // // Package adminservicemock is a generated GoMock package. diff --git a/api/historyservicemock/v1/service.pb.mock.go b/api/historyservicemock/v1/service.pb.mock.go index db052f311f..a1a21b0590 100644 --- a/api/historyservicemock/v1/service.pb.mock.go +++ b/api/historyservicemock/v1/service.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package historyservicemock -source api/historyservice/v1/service.pb.go -destination api.new/temporal/server/api/historyservicemock/v1/service.pb.mock.go +// mockgen -package historyservicemock -source api/historyservice/v1/service.pb.go -destination api/historyservicemock/v1/service.pb.mock.go // // Package historyservicemock is a generated GoMock package. diff --git a/api/historyservicemock/v1/service_grpc.pb.mock.go b/api/historyservicemock/v1/service_grpc.pb.mock.go index 083be1ff53..bb1ce12c8d 100644 --- a/api/historyservicemock/v1/service_grpc.pb.mock.go +++ b/api/historyservicemock/v1/service_grpc.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package historyservicemock -source api/historyservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/historyservicemock/v1/service_grpc.pb.mock.go +// mockgen -package historyservicemock -source api/historyservice/v1/service_grpc.pb.go -destination api/historyservicemock/v1/service_grpc.pb.mock.go // // Package historyservicemock is a generated GoMock package. diff --git a/api/matchingservicemock/v1/service.pb.mock.go b/api/matchingservicemock/v1/service.pb.mock.go index f6f4b5c2c8..abc2cf41b8 100644 --- a/api/matchingservicemock/v1/service.pb.mock.go +++ b/api/matchingservicemock/v1/service.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package matchingservicemock -source api/matchingservice/v1/service.pb.go -destination api.new/temporal/server/api/matchingservicemock/v1/service.pb.mock.go +// mockgen -package matchingservicemock -source api/matchingservice/v1/service.pb.go -destination api/matchingservicemock/v1/service.pb.mock.go // // Package matchingservicemock is a generated GoMock package. diff --git a/api/matchingservicemock/v1/service_grpc.pb.mock.go b/api/matchingservicemock/v1/service_grpc.pb.mock.go index 21aa65f868..f25c075e61 100644 --- a/api/matchingservicemock/v1/service_grpc.pb.mock.go +++ b/api/matchingservicemock/v1/service_grpc.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package matchingservicemock -source api/matchingservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/matchingservicemock/v1/service_grpc.pb.mock.go +// mockgen -package matchingservicemock -source api/matchingservice/v1/service_grpc.pb.go -destination api/matchingservicemock/v1/service_grpc.pb.mock.go // // Package matchingservicemock is a generated GoMock package. diff --git a/api/testservicemock/v1/service.pb.mock.go b/api/testservicemock/v1/service.pb.mock.go index be399d75a6..57015772c9 100644 --- a/api/testservicemock/v1/service.pb.mock.go +++ b/api/testservicemock/v1/service.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package testservicemock -source api/testservice/v1/service.pb.go -destination api.new/temporal/server/api/testservicemock/v1/service.pb.mock.go +// mockgen -package testservicemock -source api/testservice/v1/service.pb.go -destination api/testservicemock/v1/service.pb.mock.go // // Package testservicemock is a generated GoMock package. diff --git a/api/testservicemock/v1/service_grpc.pb.mock.go b/api/testservicemock/v1/service_grpc.pb.mock.go index c0119ac4b9..84b3c4a148 100644 --- a/api/testservicemock/v1/service_grpc.pb.mock.go +++ b/api/testservicemock/v1/service_grpc.pb.mock.go @@ -3,7 +3,7 @@ // // Generated by this command: // -// mockgen -package testservicemock -source api/testservice/v1/service_grpc.pb.go -destination api.new/temporal/server/api/testservicemock/v1/service_grpc.pb.mock.go +// mockgen -package testservicemock -source api/testservice/v1/service_grpc.pb.go -destination api/testservicemock/v1/service_grpc.pb.mock.go // // Package testservicemock is a generated GoMock package. diff --git a/chasm/lib/buf.yaml b/chasm/lib/buf.yaml new file mode 100644 index 0000000000..e4e6ba7107 --- /dev/null +++ b/chasm/lib/buf.yaml @@ -0,0 +1,12 @@ +# buf rules for CHASM protos +version: v1 +deps: + - buf.build/googleapis/googleapis +breaking: + use: + - WIRE +lint: + use: + - DEFAULT + except: + - PACKAGE_DIRECTORY_MATCH diff --git a/cmd/tools/protogen/main.go b/cmd/tools/protogen/main.go new file mode 100644 index 0000000000..b55b6c014f --- /dev/null +++ b/cmd/tools/protogen/main.go @@ -0,0 +1,443 @@ +package main + +import ( + "context" + "errors" + "flag" + "fmt" + "log" + "os" + "os/exec" + "path/filepath" + "regexp" + "strings" + + "github.com/fatih/color" +) + +var cyan = color.New(color.FgHiCyan, color.Bold) + +func info(format string, args ...interface{}) { + log.Println(cyan.Sprintf(format, args...)) +} + +// runCommand executes a shell command and returns an error if it fails +func runCommand(ctx context.Context, name string, args ...string) error { + cmd := exec.CommandContext(ctx, name, args...) + cmd.Stdout = os.Stdout + cmd.Stderr = os.Stderr + return cmd.Run() +} + +func copyRecursive(src, dst string) error { + srcInfo, err := os.Stat(src) + if err != nil { + return fmt.Errorf("error stating source %s: %w", src, err) + } + + if !srcInfo.IsDir() { + return fmt.Errorf("source %s is not a directory", src) + } + + err = os.MkdirAll(dst, srcInfo.Mode()) + if err != nil { + return fmt.Errorf("error creating destination directory %s: %w", dst, err) + } + + entries, err := os.ReadDir(src) + if err != nil { + return fmt.Errorf("error reading source directory %s: %w", src, err) + } + + for _, entry := range entries { + srcPath := filepath.Join(src, entry.Name()) + dstPath := filepath.Join(dst, entry.Name()) + srcInfo, err := entry.Info() + if err != nil { + return fmt.Errorf("error getting info for %s: %w", srcPath, err) + } + + if entry.IsDir() { + if err := copyRecursive(srcPath, dstPath); err != nil { + return err + } + } else { + // Fix imports in the generated mock file + content, err := os.ReadFile(srcPath) + if err != nil { + return fmt.Errorf("error reading file %s: %w", srcPath, err) + } + + if err := os.WriteFile(dstPath, []byte(content), srcInfo.Mode()); err != nil { + return fmt.Errorf("error writing file %s: %w", dstPath, err) + } + } + } + + return nil +} + +// exists checks if a path exists +func exists(path string) bool { + _, err := os.Stat(path) + return err == nil +} + +func existsDir(path string) bool { + stat, err := os.Stat(path) + return err == nil && stat.IsDir() +} + +// replaceInFile replaces text in a file using sed-like patterns +func replaceInFile(filePath, oldPattern, newPattern string) error { + content, err := os.ReadFile(filePath) + if err != nil { + return err + } + + // Convert sed pattern to Go regex + re, err := regexp.Compile(oldPattern) + if err != nil { + return err + } + + newContent := re.ReplaceAllString(string(content), newPattern) + return os.WriteFile(filePath, []byte(newContent), 0644) +} + +type generator struct { + rootDir string + tempOut string + tempProtoRoot string + protoOut string + protoBackup string + protogen string + apiBinpb string + protoRoot string + goimports string + mockgen string + chasmLibDirs []string +} + +func newGenerator() (*generator, error) { + var gen generator + flag.StringVar(&gen.protoOut, "proto-out", "", "Proto output directory (required)") + flag.StringVar(&gen.protoRoot, "proto-root", "", "Proto root directory (required)") + flag.StringVar(&gen.rootDir, "root", "", "Root directory (required)") + flag.StringVar(&gen.protogen, "protogen", "", "Path to protogen binary (required)") + flag.StringVar(&gen.apiBinpb, "api-binpb", "", "Path to API binpb file (required)") + flag.StringVar(&gen.goimports, "goimports", "", "Path to goimports binary (required)") + flag.StringVar(&gen.mockgen, "mockgen", "", "Path to mockgen binary (required)") + flag.Parse() + + // Validate required flags + if gen.protoOut == "" || gen.protoRoot == "" || gen.rootDir == "" || gen.protogen == "" || + gen.apiBinpb == "" || gen.goimports == "" || gen.mockgen == "" { + flag.Usage() + return nil, errors.New("all flags are required") + } + + chasmDir := filepath.Join(gen.rootDir, "chasm", "lib") + + ls, err := os.ReadDir(chasmDir) + if err != nil { + return nil, fmt.Errorf("error reading chasm/lib directory %s: %w", chasmDir, err) + } + gen.chasmLibDirs = make([]string, 0, len(ls)) + for _, entry := range ls { + if !entry.IsDir() { + continue + } + gen.chasmLibDirs = append(gen.chasmLibDirs, filepath.Join(chasmDir, entry.Name())) + } + gen.tempOut = gen.protoOut + ".new" + gen.tempProtoRoot = gen.protoRoot + ".tmp" + gen.protoBackup = gen.protoOut + ".old" + return &gen, nil +} + +func (g *generator) removeExistingGenDirs() error { + for _, dir := range g.chasmLibDirs { + genDir := filepath.Join(dir, "gen") + if err := os.RemoveAll(genDir); err != nil { + return fmt.Errorf("error removing directory %s: %w", genDir, err) + } + } + return nil +} + +func (g *generator) backupProtos() error { + if exists(g.protoOut) { + if err := os.Rename(g.protoOut, g.protoBackup); err != nil { + return fmt.Errorf("error backing up proto output directory %s to %s: %w", g.protoOut, g.protoBackup, err) + } + } + return nil +} + +func (g *generator) prepareTempDirs() error { + // Remove and create new directory + if err := os.RemoveAll(g.tempOut); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing directory %s: %w", g.tempOut, err) + } + if err := os.MkdirAll(g.tempOut, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %w", g.tempOut, err) + } + if err := os.RemoveAll(g.tempProtoRoot); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing directory %s: %w", g.tempOut, err) + } + if err := copyRecursive(g.protoRoot, g.tempProtoRoot); err != nil { + return fmt.Errorf("error copying proto root %s to %s: %w", g.protoRoot, g.tempProtoRoot, err) + } + return nil +} + +func (g *generator) copyChasmLibProtos() error { + for _, dir := range g.chasmLibDirs { + protoDir := filepath.Join(dir, "proto") + if !existsDir(protoDir) { + continue + } + destDir := filepath.Join(g.tempProtoRoot, "internal", "temporal", "server", "chasm", "lib", filepath.Base(dir)) + if err := os.MkdirAll(destDir, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %w", destDir, err) + } + if err := copyRecursive(protoDir, filepath.Join(destDir, "proto")); err != nil { + return fmt.Errorf("error copying proto files from %s to %s: %w", protoDir, destDir, err) + } + protos, err := filepath.Glob(filepath.Join(destDir, "proto", "**", "*.proto")) + if err != nil { + return fmt.Errorf("error finding proto files in %s: %w", destDir, err) + } + for _, proto := range protos { + if err := replaceInFile(proto, `import "chasm`, `import "temporal/server/chasm`); err != nil { + return fmt.Errorf("error updating import path in %s: %w", proto, err) + } + } + } + return nil +} + +func (g *generator) runProtogen(ctx context.Context) error { + // Run protogen + protoArgs := []string{ + "--descriptor_set_in=" + g.apiBinpb, + "--root=" + filepath.Join(g.tempProtoRoot, "internal"), + "--rewrite-enum=BuildId_State:BuildId", + "--output=" + g.tempOut, + "-p", "go-grpc_out=paths=source_relative:" + g.tempOut, + "-p", "go-helpers_out=paths=source_relative:" + g.tempOut, + } + + if err := runCommand(ctx, g.protogen, protoArgs...); err != nil { + return fmt.Errorf("error running protogen: %w", err) + } + return nil +} + +func (g *generator) runGoImports(ctx context.Context) error { + if err := runCommand(ctx, g.goimports, "-w", g.tempOut); err != nil { + return fmt.Errorf("error running goimports: %w", err) + } + + return nil +} + +func (g *generator) generateProtoMocks(ctx context.Context) error { + // Find service.pb.go and service_grpc.pb.go files + return filepath.Walk(g.tempOut, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if strings.HasSuffix(path, "service.pb.go") || strings.HasSuffix(path, "service_grpc.pb.go") { + // Convert service/ to servicemock/ and .go to .mock.go + dst := strings.ReplaceAll(path, "service/", "servicemock/") + dst = strings.ReplaceAll(dst, ".go", ".mock.go") + + // Get package name + dstDir := filepath.Dir(dst) + pkg := filepath.Base(filepath.Dir(dstDir)) + + // Create destination directory + if err := os.MkdirAll(dstDir, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %v", dstDir, err) + } + + // Run mockgen + mockgenArgs := []string{ + "-package", pkg, + "-source", path, + "-destination", dst, + } + + if err := runCommand(ctx, g.mockgen, mockgenArgs...); err != nil { + return fmt.Errorf("error running mockgen: %v", err) + } + + // Fix imports in the generated mock file + content, err := os.ReadFile(dst) + if err != nil { + return fmt.Errorf("error reading file %s: %v", dst, err) + } + + // Replace the incorrect import path + newContent := strings.ReplaceAll(string(content), g.tempOut+"/temporal/server/", "") + + if err := os.WriteFile(dst, []byte(newContent), 0644); err != nil { + return fmt.Errorf("error writing file %s: %v", dst, err) + } + } + return nil + }) +} + +func (g *generator) modifyHistoryServiceFile() error { + historyServiceFile := filepath.Join(g.tempOut, "temporal", "server", "api", "historyservice", "v1", "service_grpc.pb.go") + + // Replace GetWorkflowExecutionHistory method signature + err := replaceInFile(historyServiceFile, + `GetWorkflowExecutionHistory\(context\.Context, \*GetWorkflowExecutionHistoryRequest\) \(\*GetWorkflowExecutionHistoryResponse, error\)`, + `GetWorkflowExecutionHistory(context.Context, *GetWorkflowExecutionHistoryRequest) (*GetWorkflowExecutionHistoryResponseWithRaw, error)`) + if err != nil { + return fmt.Errorf("error modifying GetWorkflowExecutionHistory: %w", err) + } + + // Replace RecordWorkflowTaskStarted method signature + err = replaceInFile(historyServiceFile, + `RecordWorkflowTaskStarted\(context\.Context, \*RecordWorkflowTaskStartedRequest\) \(\*RecordWorkflowTaskStartedResponse, error\)`, + `RecordWorkflowTaskStarted(context.Context, *RecordWorkflowTaskStartedRequest) (*RecordWorkflowTaskStartedResponseWithRawHistory, error)`) + if err != nil { + return fmt.Errorf("error modifying RecordWorkflowTaskStarted: %w", err) + } + + return nil +} + +func (g *generator) moveProtoFiles() error { + sourceApiDir := filepath.Join(g.tempOut, "temporal", "server", "api") + return os.Rename(sourceApiDir, g.protoOut) +} + +func (g *generator) moveGeneratedChasmFiles() error { + sourceChasmDir := filepath.Join(g.tempOut, "temporal", "server", "chasm") + + return filepath.Walk(sourceChasmDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Calculate relative path from sourceChasmDir + relPath, err := filepath.Rel(sourceChasmDir, path) + if err != nil { + return err + } + // Transform relPath to match the destination structure + // e.g., "lib//proto/v1/*" -> "lib//gen/pb/*" + re := regexp.MustCompile(`^lib/([^/]+)/proto/(v\d+)/(.*)$`) + parts := re.FindStringSubmatch(relPath) + if len(parts) < 4 { + return nil + } + dstPath := filepath.Join("chasm", "lib", parts[1], "gen", parts[1]+"pb", parts[2], parts[3]) + dirName := filepath.Dir(dstPath) // Ensure the destination path is a base path + + if err := os.MkdirAll(dirName, 0755); err != nil { + return fmt.Errorf("error creating directory %s: %w", dirName, err) + } + + if err := os.Rename(path, dstPath); err != nil { + return fmt.Errorf("error moving file %s to %s: %w", path, dstPath, err) + } + return nil + }) +} + +func (g *generator) cleanup(restoreOld bool) error { + // Remove temporary directories + if err := os.RemoveAll(g.tempProtoRoot); err != nil { + return fmt.Errorf("error removing temporary proto root %s: %w", g.tempProtoRoot, err) + } + if err := os.RemoveAll(g.tempOut); err != nil { + return fmt.Errorf("error removing temporary output directory %s: %w", g.tempOut, err) + } + if restoreOld { + // Restore the old proto output directory + if exists(g.protoBackup) { + if err := os.Rename(g.protoBackup, g.protoOut); err != nil { + return fmt.Errorf("error restoring proto output directory from %s to %s: %w", g.protoBackup, g.protoOut, err) + } + } + } else { + // Remove the old proto output directory if it exists + if err := os.RemoveAll(g.protoBackup); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("error removing backup proto output directory %s: %w", g.protoBackup, err) + } + } + + return nil +} + +func main() { + ctx := context.Background() + gen, err := newGenerator() + if err != nil { + log.Fatalln("Error creating generator:", err) + } + + info("Removing existing CHASM gen directories...") + if err := gen.removeExistingGenDirs(); err != nil { + log.Fatalln("Error removing existing CHASM gen directories:", err) + } + + info("Backing up existing proto output directory...") + if err := gen.backupProtos(); err != nil { + log.Fatalln("Error backing up proto output directory:", err) + } + + genErr := generate(ctx, gen) + cleanupErr := gen.cleanup(genErr != nil) + if cleanupErr != nil { + cleanupErr = fmt.Errorf("error cleaning up after generation: %w", cleanupErr) + } + err = errors.Join(genErr, cleanupErr) + if err != nil { + log.Fatalln("Generation failed:", err) + } +} + +func generate(ctx context.Context, gen *generator) error { + info("Preparing temp directories...") + if err := gen.prepareTempDirs(); err != nil { + return fmt.Errorf("error preparing new gen directory: %w", err) + } + info("Copying CHASM lib protos...") + if err := gen.copyChasmLibProtos(); err != nil { + return fmt.Errorf("error copying CHASM lib protos: %w", err) + } + info("Running protoc for proto files...") + if err := gen.runProtogen(ctx); err != nil { + return fmt.Errorf("error running protogen: %w", err) + } + info("Running goimports for proto files...") + if err := gen.runGoImports(ctx); err != nil { + return fmt.Errorf("error running goimports: %w", err) + } + info("Generating proto mocks...") + if err := gen.generateProtoMocks(ctx); err != nil { + return fmt.Errorf("error generating mock files: %w", err) + } + info("Modifying history service server interface...") + if err := gen.modifyHistoryServiceFile(); err != nil { + return fmt.Errorf("error modifying history service file: %w", err) + } + info("Moving proto files into place...") + if err := gen.moveProtoFiles(); err != nil { + return fmt.Errorf("error moving proto files: %w", err) + } + info("Moving generated CHASM files into place...") + if err := gen.moveGeneratedChasmFiles(); err != nil { + return fmt.Errorf("error moving CHASM files: %w", err) + } + return nil +} diff --git a/develop/buf-breaking.sh b/develop/buf-breaking.sh index 80548ecb5a..b560802e86 100755 --- a/develop/buf-breaking.sh +++ b/develop/buf-breaking.sh @@ -23,6 +23,7 @@ set -eu -o pipefail : "${BUF:=buf}" : "${API_BINPB:=proto/api.binpb}" : "${INTERNAL_BINPB:=proto/image.bin}" +: "${CHASM_BINPB:=proto/chasm.bin}" : "${MAIN_BRANCH:=main}" : "${COMMIT:=$(git rev-parse HEAD)}" @@ -45,6 +46,7 @@ fi # If invoked from the Makefile, this should already be done. This is just in # case this is being run manually. $MAKE "$INTERNAL_BINPB" +$MAKE "$CHASM_BINPB" tmp=$(mktemp --tmpdir -d temporal-buf-breaking.XXXXXXXXX) trap 'rm -rf $tmp' EXIT @@ -60,7 +62,14 @@ check_against_commit() { $MAKE -C "$tmp" "$INTERNAL_BINPB" $BUF breaking "$INTERNAL_BINPB" --against "$tmp/$INTERNAL_BINPB" --config proto/internal/buf.yaml else - yellow "$name commit is too old to support breaking check" + yellow "$name commit is too old to support breaking check for internal protos" + fi + + if grep -q CHASM_BINPB "$tmp/Makefile"; then + $MAKE -C "$tmp" "$CHASM_BINPB" + $BUF breaking "$CHASM_BINPB" --against "$tmp/$CHASM_BINPB" --config chasm/lib/buf.yaml + else + yellow "$name commit is too old to support breaking check for chasm protos" fi } diff --git a/develop/protoc.sh b/develop/protoc.sh deleted file mode 100755 index fd49407020..0000000000 --- a/develop/protoc.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -set -eu -o pipefail - -color() { printf "\e[1;36m%s\e[0m\n" "$*"; } - -api=$PROTO_OUT -new=$api.new - -color "Generating protos..." - -rm -rf "$new" && mkdir -p "$new" - -$PROTOGEN \ - --descriptor_set_in="$API_BINPB" \ - --root="$PROTO_ROOT"/internal \ - --rewrite-enum=BuildId_State:BuildId \ - --output="$new" \ - -p go-grpc_out=paths=source_relative:"$new" \ - -p go-helpers_out=paths=source_relative:"$new" - -color "Run goimports for proto files..." -$GOIMPORTS -w "$new" - -color "Generate proto mocks..." -find "$new" -name service.pb.go -o -name service_grpc.pb.go | while read -r src; do - dst=$(echo "$src" | sed -e 's,service/,servicemock/,' -e 's,[.]go$,.mock.go,') - pkg=$(basename "$(dirname "$(dirname "$dst")")") - $MOCKGEN -package "$pkg" -source "$src" -destination "$dst" - # Since we're generating mocks from files in incorrect locations, mockgen - # writes incorrect imports. Fix them manually. The awkward sed/rm invocation - # is to work on linux and macos. - sed -i.replaced "s,$new/temporal/server/,," "$dst" - rm -f "$dst.replaced" -done - -color "Modify history service server interface..." -sed -i.bak \ - -e 's/GetWorkflowExecutionHistory(context\.Context, \*GetWorkflowExecutionHistoryRequest) (\*GetWorkflowExecutionHistoryResponse, error)/GetWorkflowExecutionHistory(context.Context, *GetWorkflowExecutionHistoryRequest) (*GetWorkflowExecutionHistoryResponseWithRaw, error)/g' \ - -e 's/RecordWorkflowTaskStarted(context\.Context, \*RecordWorkflowTaskStartedRequest) (\*RecordWorkflowTaskStartedResponse, error)/RecordWorkflowTaskStarted(context.Context, *RecordWorkflowTaskStartedRequest) (*RecordWorkflowTaskStartedResponseWithRawHistory, error)/g' \ - "$new"/temporal/server/api/historyservice/v1/service_grpc.pb.go -rm "$new"/temporal/server/api/historyservice/v1/service_grpc.pb.go.bak - -color "Moving proto files into place..." -old=$api.old -[[ -d "$api" ]] && mv -f "$api" "$old" -mkdir -p "$api" -mv -f "$new"/temporal/server/api/* "$api"/ -rm -rf "$new" "$old"