From 9f6a85cff7113989394255519ef1745f15b0d872 Mon Sep 17 00:00:00 2001 From: Devin Gunay Date: Thu, 27 Oct 2022 11:00:44 -0700 Subject: [PATCH] add partial ProcessBitMap example --- go/examples/processbitmap/process_bitmap.go | 105 ++++++++++++++++++++ go/go.mod | 1 + go/go.sum | 2 + 3 files changed, 108 insertions(+) create mode 100644 go/examples/processbitmap/process_bitmap.go diff --git a/go/examples/processbitmap/process_bitmap.go b/go/examples/processbitmap/process_bitmap.go new file mode 100644 index 0000000..7a6745f --- /dev/null +++ b/go/examples/processbitmap/process_bitmap.go @@ -0,0 +1,105 @@ +package changeeventheader + +import ( + "context" + "encoding/hex" + "fmt" + "strings" + + "github.com/boljen/go-bitmap" +) + +var ErrUnexpectedType = fmt.Errorf("unexpected type") + +// ProcessBitMap takes the bitmap-encoded list of changed fields and returns a list of field names. +// See Salesforce pub-sub docs for more details: +// https://developer.salesforce.com/docs/platform/pub-sub-api/guide/event-deserialization-considerations.html +func ProcessBitMap(ctx context.Context, avroSchema map[string]any, bitmapFields []string) ([]string, error) { + changedFieldNames := []string{} + if len(bitmapFields) == 0 { + return changedFieldNames, nil + } + + // Replace top field level bitmap with list of fields + if strings.HasPrefix(bitmapFields[0], "0x") { + // Convert hex to bytes + hexStr := bitmapFields[0][2:] + bytes, err := hex.DecodeString(hexStr) + if err != nil { + return nil, fmt.Errorf("decode hex string '%s': %w", hexStr, err) + } + + // Reverse to little-endian + reversedBytes := []byte{} + for i := len(bytes) - 1; i >= 0; i-- { + reversedBytes = append(reversedBytes, bytes[i]) + } + + bitMap := bitmap.Bitmap(reversedBytes) + + schemaFieldNames, err := getSchemaFieldNames(avroSchema) + if err != nil { + return nil, fmt.Errorf("get schema field names: %w", err) + } + + changedFieldNames = append( + changedFieldNames, + getFieldNamesFromBitString(bitMap, schemaFieldNames)..., + ) + bitmapFields = bitmapFields[1:] // shift off the front + } + + // There can be other bitmaps present in the message of the form: + // parentPos-childBitMap + // If we end up needing that, we can implement it here. + // Check the example repository for Python or Java examples (no Go examples exist yet). + + return changedFieldNames, nil +} + +func getSchemaFieldNames(schema map[string]any) ([]string, error) { + fields := []string{} + fieldsSlice, ok := schema["fields"].([]any) + if !ok { + return nil, ErrCouldNotConvertType(schema["fields"], []any{}) + } + + for _, fieldObj := range fieldsSlice { + field, ok := fieldObj.(map[string]any) + if !ok { + return nil, ErrCouldNotConvertType(fieldObj, map[string]any{}) + } + + asString, ok := field["name"].(string) + if !ok { + return nil, ErrCouldNotConvertType(field["name"], "") + } + fields = append(fields, asString) + } + return fields, nil +} + +func ErrCouldNotConvertType(from any, to any) error { + return fmt.Errorf("could not convert %T to type %T: %w", from, to, ErrUnexpectedType) +} + +func getFieldNamesFromBitString( + bitMap bitmap.Bitmap, + // A list of all of the schema field names + schemaFieldNames []string, +) []string { + // Find indices of "1" bits + oneIndices := []int{} + for i := 0; i < bitMap.Len(); i++ { + if bitMap.Get(i) { + oneIndices = append(oneIndices, i) + } + } + + // And then pick the schema field names with those indices + changedFieldNames := []string{} + for _, index := range oneIndices { + changedFieldNames = append(changedFieldNames, schemaFieldNames[index]) + } + return changedFieldNames +} diff --git a/go/go.mod b/go/go.mod index 485001b..46fa7b4 100644 --- a/go/go.mod +++ b/go/go.mod @@ -3,6 +3,7 @@ module github.com/developerforce/pub-sub-api/go go 1.18 require ( + github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d github.com/linkedin/goavro/v2 v2.11.0 google.golang.org/grpc v1.37.0 google.golang.org/protobuf v1.26.0 diff --git a/go/go.sum b/go/go.sum index 6038168..2f0cdc0 100644 --- a/go/go.sum +++ b/go/go.sum @@ -1,5 +1,7 @@ cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d h1:zsO4lp+bjv5XvPTF58Vq+qgmZEYZttJK+CWtSZhKenI= +github.com/boljen/go-bitmap v0.0.0-20151001105940-23cd2fb0ce7d/go.mod h1:f1iKL6ZhUWvbk7PdWVmOaak10o86cqMUYEmn1CZNGEI= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=