APIGOV-26411 - update metric reporting (#150)

* update deps * add allTraffic config * update job start process * clean unused defs * fix tests * directly report metrics and publish at end of interval * add ability to report not set * update tests * update deps * cleanup processing * cleanup * update deps and test * update deps
Axway · Sep 29, 2023 · 04dc252 · 04dc252
1 parent 6883c38
commit 04dc252
Show file tree

Hide file tree

Showing 21 changed files with 1,073 additions and 1,443 deletions.
diff --git a/client/go.mod b/client/go.mod
@@ -4,7 +4,7 @@ go 1.18
 
 // replace github.com/Axway/agent-sdk => /home/ubuntu/go/src/github.com/Axway/agent-sdk
 
-require github.com/Axway/agent-sdk v1.1.63
+require github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2
 
 require (
 	github.com/armon/go-radix v1.0.0 // indirect

diff --git a/client/go.sum b/client/go.sum
@@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Axway/agent-sdk v1.1.63 h1:4ozpjwU8pqgyBEJ40jzA5rpvKpV69pQ9jMgeVK3ywdk=
-github.com/Axway/agent-sdk v1.1.63/go.mod h1:IjZ/VyF0fQezmrxSRI7l9jEuOGYl8Hc/XkotLuIYcnw=
+github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2 h1:z7JCNXTw9TgES88sh+hFOZ1lOZZLcqkvnMaUIm5lm88=
+github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2/go.mod h1:IjZ/VyF0fQezmrxSRI7l9jEuOGYl8Hc/XkotLuIYcnw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

diff --git a/client/pkg/config/config.go b/client/pkg/config/config.go
@@ -19,9 +19,11 @@ type ApigeeConfig struct {
 	Filter          string           `config:"filter"`
 	DeveloperID     string           `config:"developerID"`
 	Auth            *AuthConfig      `config:"auth"`
-	Intervals       *ApigeeIntervals `config:"intervals"`
+	Intervals       *ApigeeIntervals `config:"interval"`
 	Workers         *ApigeeWorkers   `config:"workers"`
 	CloneAttributes bool             `config:"cloneAttributes"`
+	AllTraffic      bool             `config:"allTraffic"`
+	NotSetTraffic   bool             `config:"notSetTraffic"`
 	mode            discoveryMode
 }
 
@@ -77,6 +79,8 @@ const (
 	pathMode               = "apigee.discoveryMode"
 	pathFilter             = "apigee.filter"
 	pathCloneAttributes    = "apigee.cloneAttributes"
+	pathAllTraffic         = "apigee.allTraffic"
+	pathNotSetTraffic      = "apigee.notSetTraffic"
 	pathAuthURL            = "apigee.auth.url"
 	pathAuthServerUsername = "apigee.auth.serverUsername"
 	pathAuthServerPassword = "apigee.auth.serverPassword"
@@ -103,7 +107,9 @@ func AddProperties(rootProps properties.Properties) {
 	rootProps.AddStringProperty(pathAuthURL, "https://login.apigee.com", "URL to use when authenticating to APIGEE")
 	rootProps.AddStringProperty(pathAuthServerUsername, "edgecli", "Username to use to when requesting APIGEE token")
 	rootProps.AddStringProperty(pathAuthServerPassword, "edgeclisecret", "Password to use to when requesting APIGEE token")
-	rootProps.AddBoolProperty(pathCloneAttributes, false, "Set to true to copy the tags when provisioning a Product in product mode.")
+	rootProps.AddBoolProperty(pathCloneAttributes, false, "Set to true to copy the tags when provisioning a Product in product mode")
+	rootProps.AddBoolProperty(pathAllTraffic, false, "Set to true to report metrics for all traffic for the selected mode")
+	rootProps.AddBoolProperty(pathNotSetTraffic, false, "Set to true to report metrics for values reported with (not set) ast the name")
 	rootProps.AddStringProperty(pathAuthUsername, "", "Username to use to authenticate to APIGEE")
 	rootProps.AddStringProperty(pathAuthPassword, "", "Password for the user to authenticate to APIGEE")
 	rootProps.AddDurationProperty(pathSpecInterval, 30*time.Minute, "The time interval between checking for updated specs", properties.WithLowerLimit(1*time.Minute))
@@ -127,6 +133,8 @@ func ParseConfig(rootProps properties.Properties) *ApigeeConfig {
 		mode:            stringToDiscoveryMode(rootProps.StringPropertyValue(pathMode)),
 		Filter:          rootProps.StringPropertyValue(pathFilter),
 		CloneAttributes: rootProps.BoolPropertyValue(pathCloneAttributes),
+		AllTraffic:      rootProps.BoolPropertyValue(pathAllTraffic),
+		NotSetTraffic:   rootProps.BoolPropertyValue(pathNotSetTraffic),
 		Intervals: &ApigeeIntervals{
 			Stats:   rootProps.DurationPropertyValue(pathStatsInterval),
 			Proxy:   rootProps.DurationPropertyValue(pathProxyInterval),
@@ -215,3 +223,11 @@ func (a *ApigeeConfig) IsProductMode() bool {
 func (a *ApigeeConfig) ShouldCloneAttributes() bool {
 	return a.CloneAttributes
 }
+
+func (a *ApigeeConfig) ShouldReportAllTraffic() bool {
+	return a.AllTraffic
+}
+
+func (a *ApigeeConfig) ShouldReportNotSetTraffic() bool {
+	return a.NotSetTraffic
+}
diff --git a/discovery/go.mod b/discovery/go.mod
@@ -5,7 +5,7 @@ go 1.18
 // replace github.com/Axway/agent-sdk => /home/ubuntu/go/src/github.com/Axway/agent-sdk
 
 require (
-	github.com/Axway/agent-sdk v1.1.63
+	github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2
 	github.com/Axway/agents-apigee/client v0.0.0-00010101000000-000000000000
 	github.com/stretchr/testify v1.8.4
 )

diff --git a/discovery/go.sum b/discovery/go.sum
@@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Axway/agent-sdk v1.1.63 h1:4ozpjwU8pqgyBEJ40jzA5rpvKpV69pQ9jMgeVK3ywdk=
-github.com/Axway/agent-sdk v1.1.63/go.mod h1:IjZ/VyF0fQezmrxSRI7l9jEuOGYl8Hc/XkotLuIYcnw=
+github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2 h1:z7JCNXTw9TgES88sh+hFOZ1lOZZLcqkvnMaUIm5lm88=
+github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2/go.mod h1:IjZ/VyF0fQezmrxSRI7l9jEuOGYl8Hc/XkotLuIYcnw=
 github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=

diff --git a/discovery/pkg/apigee/pollproxiesjob.go b/discovery/pkg/apigee/pollproxiesjob.go
@@ -374,6 +374,10 @@ func (j *pollProxiesJob) publish(ctx context.Context) {
 		logger.WithError(err).Error("building service body")
 		return
 	}
+	if serviceBody == nil {
+		return
+	}
+
 	serviceBodyHash, _ := coreutil.ComputeHash(*serviceBody)
 	hashString := util.ConvertUnitToString(serviceBodyHash)
 	cacheKey := createProxyCacheKey(getStringFromContext(ctx, proxyNameField), envName)
@@ -420,7 +424,8 @@ func (j *pollProxiesJob) buildServiceBody(ctx context.Context) (*apic.ServiceBod
 	}
 
 	if len(spec) == 0 {
-		return nil, fmt.Errorf("skipping proxy creation without a spec")
+		log.Warn("skipping proxy creation without a spec")
+		return nil, nil
 	}
 	logger.Debug("creating service body")
 

diff --git a/traceability/apigee_traceability_agent.yml b/traceability/apigee_traceability_agent.yml
@@ -6,8 +6,10 @@ apigee_traceability_agent:
     organization: ${APIGEE_ORGANIZATION}
     developerID: ${APIGEE_DEVELOPERID}
     discoveryMode: ${APIGEE_DISCOVERYMODE}
-    intervals:
-      stats: ${APIGEE_INTERVAL_STATS}
+    allTraffic: ${APIGEE_ALLTRAFFIC}
+    notSetTraffic: ${APIGEE_NOTSETTRAFFIC}
+    interval:
+      stats: ${APIGEE_INTERVAL_STATS:5m}
     auth:
       username: ${APIGEE_AUTH_USERNAME}
       password: ${APIGEE_AUTH_PASSWORD}
@@ -30,6 +32,7 @@ apigee_traceability_agent:
       publish: ${CENTRAL_USAGEREPORTING_PUBLISH}
       publishMetric: ${CENTRAL_USAGEREPORTING_PUBLISHMETRIC}
       interval: ${CENTRAL_USAGEREPORTING_INTERVAL}
+      reportInterval: ${CENTRAL_USAGEREPORTING_REPORTINTERVAL}
       usageSchedule: ${CENTRAL_USAGEREPORTING_USAGESCHEDULE}
       offline: ${CENTRAL_USAGEREPORTING_OFFLINE}
       offlineSchedule: ${CENTRAL_USAGEREPORTING_OFFLINESCHEDULE}

diff --git a/traceability/go.mod b/traceability/go.mod
@@ -7,7 +7,7 @@ go 1.18
 replace github.com/Axway/agents-apigee/client => ../client
 
 require (
-	github.com/Axway/agent-sdk v1.1.63
+	github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2
 	github.com/Axway/agents-apigee/client v0.0.0-00010101000000-000000000000
 	github.com/elastic/beats/v7 v7.17.5
 	github.com/gofrs/uuid v4.2.0+incompatible

diff --git a/traceability/go.sum b/traceability/go.sum
@@ -36,8 +36,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
-github.com/Axway/agent-sdk v1.1.63 h1:4ozpjwU8pqgyBEJ40jzA5rpvKpV69pQ9jMgeVK3ywdk=
-github.com/Axway/agent-sdk v1.1.63/go.mod h1:IjZ/VyF0fQezmrxSRI7l9jEuOGYl8Hc/XkotLuIYcnw=
+github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2 h1:z7JCNXTw9TgES88sh+hFOZ1lOZZLcqkvnMaUIm5lm88=
+github.com/Axway/agent-sdk v1.1.64-0.20230929170957-5f398c5338b2/go.mod h1:IjZ/VyF0fQezmrxSRI7l9jEuOGYl8Hc/XkotLuIYcnw=
 github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8=
 github.com/Azure/go-autorest v12.2.0+incompatible/go.mod h1:r+4oMnoxhatjLLJ6zxSWATqVooLgysK6ZNox3g/xq24=
 github.com/Azure/go-autorest/autorest v0.11.12/go.mod h1:eipySxLmqSyC5s5k1CLupqet0PSENBEDP93LQ9a8QYw=

diff --git a/traceability/pkg/apigee/agent.go b/traceability/pkg/apigee/agent.go
@@ -91,31 +91,42 @@ func (a *Agent) registerPollStatsJob() (string, error) {
 		withIsReady(a.IsReady),
 		withStatsCache(a.statCache),
 		withCachePath(a.cacheFilePath),
+		withAllTraffic(a.cfg.ApigeeCfg.ShouldReportAllTraffic()),
+		withNotSetTraffic(a.cfg.ApigeeCfg.ShouldReportNotSetTraffic()),
 	}
-	if a.apigeeClient.GetConfig().IsProductMode() {
+	if a.cfg.ApigeeCfg.IsProductMode() {
 		baseOpts = append(baseOpts, withProductMode())
 	}
 
-	job := newPollStatsJob(append(baseOpts, withCacheClean())...)
 	lastStatTimeIface, err := a.statCache.Get(lastStartTimeKey)
+	var lastStartTime time.Time
 	if err == nil {
-		//"2022-01-21T11:31:32.079962632-07:00"
 		// there was a last time in the cache
-		lastStatTime, _ := time.Parse(time.RFC3339Nano, lastStatTimeIface.(string))
-		if time.Now().Add(time.Hour*-1).Sub(lastStatTime) > 0 {
-			// last start time not within an hour
-			catchUpJob := newPollStatsJob(
-				append(baseOpts,
-					[]func(*pollApigeeStats){
-						withStartTime(lastStatTime),
-						withIncrement(time.Hour),
-					}...,
-				)...,
-			) // create the job that catches up and stops
-			catchUpJob.Execute()
-		}
+		lastStartTime, _ = time.Parse(time.RFC3339Nano, lastStatTimeIface.(string))
+	}
+
+	if !lastStartTime.IsZero() {
+		// last start time not zero
+
+		// create the job that executes once to get all the stats that were missed
+		catchUpJob := newPollStatsJob(
+			append(baseOpts,
+				withStartTime(lastStartTime),
+			)...,
+		)
+		catchUpJob.Execute()
+
+		// register the regular running job after one interval has passed
+		go func() {
+			time.Sleep(a.cfg.ApigeeCfg.Intervals.Stats)
+			job := newPollStatsJob(append(baseOpts, withCacheClean(), withStartTime(catchUpJob.startTime))...)
+			jobs.RegisterIntervalJobWithName(job, a.cfg.ApigeeCfg.Intervals.Stats, "Apigee Stats")
+		}()
+	} else {
+		// register a regular running job, only grabbing hte last hour of stats
+		job := newPollStatsJob(append(baseOpts, withCacheClean(), withStartTime(time.Now().Add(time.Hour*-1).Truncate(time.Minute)))...)
+		jobs.RegisterIntervalJobWithName(job, a.cfg.ApigeeCfg.Intervals.Stats, "Apigee Stats")
 	}
 
-	jobs.RegisterIntervalJobWithName(job, a.cfg.ApigeeCfg.Intervals.Stats, "Apigee Stats")
 	return "", nil
 }
diff --git a/traceability/pkg/apigee/definitions.go b/traceability/pkg/apigee/definitions.go
@@ -9,19 +9,3 @@ type AuthResponse struct {
 	Scope        string `json:"scope"`
 	JTI          string `json:"jti"`
 }
-
-// Headers - Type for request/response headers
-type Headers map[string]string
-
-type metricCache struct {
-	ProxyName           string `json:"proxy"`
-	Timestamp           int64  `json:"timestamp"`
-	Total               int    `json:"total"`
-	PolicyError         int    `json:"policyError"`
-	ServerError         int    `json:"serverError"`
-	Success             int    `json:"success"`
-	ReportedPolicyError int    `json:"reportedPolicyError"`
-	ReportedServerError int    `json:"reportedServerError"`
-	ReportedSuccess     int    `json:"reportedSuccess"`
-	ResponseTime        int64  `json:"responseTime"`
-}