diff --git a/main.go b/main.go index 735a76f1..a6b47c47 100644 --- a/main.go +++ b/main.go @@ -35,7 +35,7 @@ func main() { func run(_ *cli.Context) error { ctx := signals.SetupSignalContext() debugconfig.MustSetupDebug() - s, err := config.ToServer(ctx, false) + s, err := config.ToServer(ctx, debugconfig.SQLCache) if err != nil { return err } diff --git a/pkg/debug/cli.go b/pkg/debug/cli.go index a3e9bbdb..13021d58 100644 --- a/pkg/debug/cli.go +++ b/pkg/debug/cli.go @@ -13,6 +13,7 @@ import ( type Config struct { Debug bool DebugLevel int + SQLCache bool } func (c *Config) MustSetupDebug() { @@ -54,6 +55,10 @@ func Flags(config *Config) []cli.Flag { Value: 7, Destination: &config.DebugLevel, }, + cli.BoolFlag{ + Name: "sql-cache", + Destination: &config.SQLCache, + }, } } @@ -68,5 +73,9 @@ func FlagsV2(config *Config) []cliv2.Flag { Value: 7, Destination: &config.DebugLevel, }, + &cliv2.BoolFlag{ + Name: "sql-cache", + Destination: &config.SQLCache, + }, } } diff --git a/pkg/resources/virtual/common/common.go b/pkg/resources/virtual/common/common.go index 97fa7778..3d072ad8 100644 --- a/pkg/resources/virtual/common/common.go +++ b/pkg/resources/virtual/common/common.go @@ -10,7 +10,6 @@ import ( wranglerSummary "github.com/rancher/wrangler/v3/pkg/summary" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/client-go/tools/cache" ) // SummaryCache provides an interface to get a summary/relationships for an object. Implemented by the summaryCache @@ -24,26 +23,14 @@ type DefaultFields struct { Cache SummaryCache } -// GetTransform produces the default transformation func -func (d *DefaultFields) GetTransform() cache.TransformFunc { - return d.transform -} - -// transform implements virtual.VirtualTransformFunc, and adds reserved fields/summary -func (d *DefaultFields) transform(obj any) (any, error) { - raw, isSignal, err := getUnstructured(obj) - if isSignal { - return obj, nil - } - if err != nil { - return nil, err - } - raw = addIDField(raw) - raw, err = addSummaryFields(raw, d.Cache) +// TransformCommon implements virtual.VirtualTransformFunc, and adds reserved fields/summary +func (d *DefaultFields) TransformCommon(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + obj = addIDField(obj) + obj, err := addSummaryFields(obj, d.Cache) if err != nil { return nil, fmt.Errorf("unable to add summary fields: %w", err) } - return raw, nil + return obj, nil } // addSummaryFields adds the virtual fields for object state. diff --git a/pkg/resources/virtual/common/common_test.go b/pkg/resources/virtual/common/common_test.go index 78141d57..8028f363 100644 --- a/pkg/resources/virtual/common/common_test.go +++ b/pkg/resources/virtual/common/common_test.go @@ -9,11 +9,10 @@ import ( "github.com/stretchr/testify/require" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/client-go/tools/cache" ) -func TestTransform(t *testing.T) { +func TestTransformCommonObjects(t *testing.T) { tests := []struct { name string input any @@ -160,29 +159,29 @@ func TestTransform(t *testing.T) { } for _, test := range tests { t.Run(test.name, func(t *testing.T) { - fakeCache := fakeSummaryCache{ - summarizedObject: test.hasSummary, - relationships: test.hasRelationships, + fakeCache := common.FakeSummaryCache{ + SummarizedObject: test.hasSummary, + Relationships: test.hasRelationships, } df := common.DefaultFields{ Cache: &fakeCache, } - output, err := df.GetTransform()(test.input) - require.Equal(t, test.wantOutput, output) + raw, isSignal, err := common.GetUnstructured(test.input) + if err != nil { + require.True(t, test.wantError) + return + } + if isSignal { + require.Equal(t, test.input, test.wantOutput) + return + } + output, err := df.TransformCommon(raw) if test.wantError { require.Error(t, err) } else { + require.Equal(t, test.wantOutput, output) require.NoError(t, err) } }) } } - -type fakeSummaryCache struct { - summarizedObject *summary.SummarizedObject - relationships []summarycache.Relationship -} - -func (f *fakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) { - return f.summarizedObject, f.relationships -} diff --git a/pkg/resources/virtual/common/testutil.go b/pkg/resources/virtual/common/testutil.go new file mode 100644 index 00000000..01310c51 --- /dev/null +++ b/pkg/resources/virtual/common/testutil.go @@ -0,0 +1,16 @@ +package common + +import ( + "github.com/rancher/steve/pkg/summarycache" + "github.com/rancher/wrangler/v3/pkg/summary" + "k8s.io/apimachinery/pkg/runtime" +) + +type FakeSummaryCache struct { + SummarizedObject *summary.SummarizedObject + Relationships []summarycache.Relationship +} + +func (f *FakeSummaryCache) SummaryAndRelationship(runtime.Object) (*summary.SummarizedObject, []summarycache.Relationship) { + return f.SummarizedObject, f.Relationships +} diff --git a/pkg/resources/virtual/common/util.go b/pkg/resources/virtual/common/util.go index fb496ea8..8659d868 100644 --- a/pkg/resources/virtual/common/util.go +++ b/pkg/resources/virtual/common/util.go @@ -11,7 +11,7 @@ import ( // GetUnstructured retrieves an unstructured object from the provided input. If this is a signal // object (like cache.DeletedFinalStateUnknown), returns true, indicating that this wasn't an // unstructured object, but doesn't need to be processed by our transform function -func getUnstructured(obj any) (*unstructured.Unstructured, bool, error) { +func GetUnstructured(obj any) (*unstructured.Unstructured, bool, error) { raw, ok := obj.(*unstructured.Unstructured) if !ok { _, isFinalUnknown := obj.(cache.DeletedFinalStateUnknown) diff --git a/pkg/resources/virtual/events/events.go b/pkg/resources/virtual/events/events.go new file mode 100644 index 00000000..1b6ed5d3 --- /dev/null +++ b/pkg/resources/virtual/events/events.go @@ -0,0 +1,16 @@ +// Package common provides cache.TransformFunc's for /v1 Event objects +package events + +import ( + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +// TransformEventObject does special-case handling on event objects +// 1. (only one so far): replaces the _type field with the contents of the field named "type", if it exists +func TransformEventObject(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + currentTypeValue, ok := obj.Object["type"] + if ok { + obj.Object["_type"] = currentTypeValue + } + return obj, nil +} diff --git a/pkg/resources/virtual/events/events_test.go b/pkg/resources/virtual/events/events_test.go new file mode 100644 index 00000000..72e673b1 --- /dev/null +++ b/pkg/resources/virtual/events/events_test.go @@ -0,0 +1,93 @@ +package events_test + +import ( + "testing" + + "github.com/rancher/steve/pkg/resources/virtual/events" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestTransformEvents(t *testing.T) { + tests := []struct { + name string + input any + wantOutput any + wantError bool + }{ + { + name: "fix event fields", + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "gregsFarm", + "namespace": "gregsNamespace", + }, + "id": "eventTest1id", + "type": "Gorniplatz", + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "gregsFarm", + "namespace": "gregsNamespace", + }, + "id": "eventTest1id", + "type": "Gorniplatz", + "_type": "Gorniplatz", + }, + }, + }, + { + name: "don't fix non-default-group event fields", + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "palau.io/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "gregsFarm", + "namespace": "gregsNamespace", + }, + "id": "eventTest1id", + "type": "Gorniplatz", + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "palau.io/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "gregsFarm", + "namespace": "gregsNamespace", + }, + "id": "eventTest1id", + "type": "Gorniplatz", + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + var output interface{} + var err error + raw, ok := test.input.(*unstructured.Unstructured) + if ok && raw.GetKind() == "Event" && raw.GetAPIVersion() == "/v1" { + output, err = events.TransformEventObject(raw) + } else { + output = raw + err = nil + } + require.Equal(t, test.wantOutput, output) + if test.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/pkg/resources/virtual/virtual.go b/pkg/resources/virtual/virtual.go index ec297e31..270b786b 100644 --- a/pkg/resources/virtual/virtual.go +++ b/pkg/resources/virtual/virtual.go @@ -3,7 +3,11 @@ package virtual import ( + "fmt" + "github.com/rancher/steve/pkg/resources/virtual/common" + "github.com/rancher/steve/pkg/resources/virtual/events" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/cache" ) @@ -16,13 +20,36 @@ type TransformBuilder struct { // NewTransformBuilder returns a TransformBuilder using the given summary cache func NewTransformBuilder(cache common.SummaryCache) *TransformBuilder { return &TransformBuilder{ - &common.DefaultFields{ + defaultFields: &common.DefaultFields{ Cache: cache, }, } } -// GetTransformFunc retrieves a TransformFunc for a given GVK. Currently only returns a transformFunc for defaultFields -func (t *TransformBuilder) GetTransformFunc(_ schema.GroupVersionKind) cache.TransformFunc { - return t.defaultFields.GetTransform() +// GetTransformFunc returns the func to transform a raw object into a fixed object, if needed +func (t *TransformBuilder) GetTransformFunc(gvk schema.GroupVersionKind) cache.TransformFunc { + converters := make([]func(*unstructured.Unstructured) (*unstructured.Unstructured, error), 0) + if gvk.Kind == "Event" && gvk.Group == "" && gvk.Version == "v1" { + converters = append(converters, events.TransformEventObject) + } + converters = append(converters, t.defaultFields.TransformCommon) + + return func(raw interface{}) (interface{}, error) { + obj, isSignal, err := common.GetUnstructured(raw) + if isSignal { + // isSignal= true overrides any error + return raw, err + } + if err != nil { + return nil, fmt.Errorf("GetUnstructured: failed to get underlying object: %w", err) + } + // Conversions are run in this loop: + for _, f := range converters { + obj, err = f(obj) + if err != nil { + return nil, err + } + } + return obj, nil + } } diff --git a/pkg/resources/virtual/virtual_test.go b/pkg/resources/virtual/virtual_test.go new file mode 100644 index 00000000..f9f05d27 --- /dev/null +++ b/pkg/resources/virtual/virtual_test.go @@ -0,0 +1,202 @@ +package virtual_test + +import ( + "github.com/rancher/steve/pkg/resources/virtual" + "k8s.io/apimachinery/pkg/runtime/schema" + "strings" + "testing" + + "github.com/rancher/steve/pkg/resources/virtual/common" + "github.com/rancher/steve/pkg/summarycache" + "github.com/rancher/wrangler/v3/pkg/summary" + "github.com/stretchr/testify/require" + v1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" +) + +func TestTransformChain(t *testing.T) { + tests := []struct { + name string + input any + hasSummary *summary.SummarizedObject + hasRelationships []summarycache.Relationship + wantOutput any + wantError bool + }{ + { + name: "add summary + relationships + reserved fields", + hasSummary: &summary.SummarizedObject{ + PartialObjectMetadata: v1.PartialObjectMetadata{ + ObjectMeta: v1.ObjectMeta{ + Name: "testobj", + Namespace: "test-ns", + }, + TypeMeta: v1.TypeMeta{ + APIVersion: "test.cattle.io/v1", + Kind: "TestResource", + }, + }, + Summary: summary.Summary{ + State: "success", + Transitioning: false, + Error: false, + Message: []string{"resource 1 rolled out", "resource 2 rolled out"}, + }, + }, + hasRelationships: []summarycache.Relationship{ + { + ToID: "1345", + ToType: "SomeType", + ToNamespace: "some-ns", + FromID: "78901", + FromType: "TestResource", + Rel: "uses", + }, + }, + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + }, + "id": "old-id", + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cattle.io/v1", + "kind": "TestResource", + "metadata": map[string]interface{}{ + "name": "testobj", + "namespace": "test-ns", + "state": map[string]interface{}{ + "name": "success", + "error": false, + "transitioning": false, + "message": "resource 1 rolled out:resource 2 rolled out", + }, + "relationships": []any{ + map[string]any{ + "toId": "1345", + "toType": "SomeType", + "toNamespace": "some-ns", + "fromId": "78901", + "fromType": "TestResource", + "rel": "uses", + }, + }, + }, + "id": "test-ns/testobj", + "_id": "old-id", + }, + }, + }, + { + name: "processable event", + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "oswaldsFarm", + "namespace": "oswaldsNamespace", + }, + "status": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "status": "False", + "reason": "Error", + "message": "some error", + "lastTransitionTime": "2024-01-01", + }, + }, + }, + "id": "eventTest2id", + "type": "Gorniplatz", + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "oswaldsFarm", + "namespace": "oswaldsNamespace", + "relationships": []any(nil), + }, + "status": map[string]interface{}{ + "conditions": []interface{}{ + map[string]interface{}{ + "status": "False", + "reason": "Error", + "transitioning": false, + "error": true, + "message": "some error", + "lastTransitionTime": "2024-01-01", + "lastUpdateTime": "2024-01-01", + }, + }, + }, + "id": "oswaldsNamespace/oswaldsFarm", + "_id": "eventTest2id", + "type": "Gorniplatz", + "_type": "Gorniplatz", + }, + }, + }, + { + name: "don't fix non-default-group event fields", + input: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "palau.io/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "gregsFarm", + "namespace": "gregsNamespace", + "relationships": []any(nil), + }, + "id": "eventTest1id", + "type": "Gorniplatz", + }, + }, + wantOutput: &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "palau.io/v1", + "kind": "Event", + "metadata": map[string]interface{}{ + "name": "gregsFarm", + "namespace": "gregsNamespace", + "relationships": []any(nil), + }, + "id": "gregsNamespace/gregsFarm", + "_id": "eventTest1id", + "type": "Gorniplatz", + }, + }, + }, + } + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + fakeCache := common.FakeSummaryCache{ + SummarizedObject: test.hasSummary, + Relationships: test.hasRelationships, + } + tb := virtual.NewTransformBuilder(&fakeCache) + raw, isSignal, err := common.GetUnstructured(test.input) + require.False(t, isSignal) + require.Nil(t, err) + apiVersion := raw.GetAPIVersion() + parts := strings.Split(apiVersion, "/") + gvk := schema.GroupVersionKind{Group: parts[0], Version: parts[1], Kind: raw.GetKind()} + output, err := tb.GetTransformFunc(gvk)(test.input) + require.Equal(t, test.wantOutput, output) + if test.wantError { + require.Error(t, err) + } else { + require.NoError(t, err) + } + }) + } +} diff --git a/pkg/stores/sqlproxy/proxy_store.go b/pkg/stores/sqlproxy/proxy_store.go index d2ab50f6..a6324a32 100644 --- a/pkg/stores/sqlproxy/proxy_store.go +++ b/pkg/stores/sqlproxy/proxy_store.go @@ -59,12 +59,47 @@ var ( paramScheme = runtime.NewScheme() paramCodec = runtime.NewParameterCodec(paramScheme) typeSpecificIndexedFields = map[string][][]string{ - "_v1_Namespace": {{`metadata`, `labels[field.cattle.io/projectId]`}}, - "_v1_Node": {{`status`, `nodeInfo`, `kubeletVersion`}, {`status`, `nodeInfo`, `operatingSystem`}}, - "_v1_Pod": {{`spec`, `containers`, `image`}, {`spec`, `nodeName`}}, - "_v1_ConfigMap": {{`metadata`, `labels[harvesterhci.io/cloud-init-template]`}}, - - "management.cattle.io_v3_Node": {{`status`, `nodeName`}}, + gvkKey("", "v1", "Event"): { + {"_type"}, + {"involvedObject", "kind"}, + {"message"}, + {"reason"}, + }, + gvkKey("", "v1", "Namespace"): { + {"metadata", "labels[field.cattle.io/projectId]"}}, + gvkKey("", "v1", "Node"): { + {"status", "nodeInfo", "kubeletVersion"}, + {"status", "nodeInfo", "operatingSystem"}}, + gvkKey("", "v1", "Pod"): { + {"spec", "containers", "image"}, + {"spec", "nodeName"}}, + gvkKey("", "v1", "ConfigMap"): { + {"metadata", "labels[harvesterhci.io/cloud-init-template]"}}, + gvkKey("catalog.cattle.io", "v1", "ClusterRepo"): { + {"metadata", "annotations[clusterrepo.cattle.io/hidden]"}, + {"spec", "gitBranch"}, + {"spec", "gitRepo"}, + }, + gvkKey("catalog.cattle.io", "v1", "Operation"): { + {"status", "action"}, + {"status", "namespace"}, + {"status", "releaseName"}, + }, + gvkKey("cluster.x-k8s.io", "v1beta1", "Machine"): { + {"spec", "clusterName"}}, + gvkKey("management.cattle.io", "v3", "Node"): { + {"status", "nodeName"}}, + gvkKey("management.cattle.io", "v3", "NodePool"): { + {"spec", "clusterName"}}, + gvkKey("management.cattle.io", "v3", "NodeTemplate"): { + {"spec", "clusterName"}}, + gvkKey("provisioning.cattle.io", "v1", "Cluster"): { + {"metadata", "labels[provider.cattle.io]"}, + {"status", "provider"}, + {"status", "allocatable", "cpu"}, + {"status", "allocatable", "memory"}, + {"status", "allocatable", "pods"}, + }, } commonIndexFields = [][]string{ {`id`}, @@ -239,15 +274,15 @@ func (s *Store) initializeNamespaceCache() error { func getFieldForGVK(gvk schema.GroupVersionKind) [][]string { fields := [][]string{} fields = append(fields, commonIndexFields...) - typeFields := typeSpecificIndexedFields[keyFromGVK(gvk)] + typeFields := typeSpecificIndexedFields[gvkKey(gvk.Group, gvk.Version, gvk.Kind)] if typeFields != nil { fields = append(fields, typeFields...) } return fields } -func keyFromGVK(gvk schema.GroupVersionKind) string { - return gvk.Group + "_" + gvk.Version + "_" + gvk.Kind +func gvkKey(group, version, kind string) string { + return group + "_" + version + "_" + kind } // getFieldsFromSchema converts object field names from types.APISchema's format into lasso's