From c6f9da0e61594e863301ff53f72d9ffdfe6e9bc1 Mon Sep 17 00:00:00 2001 From: Peiman Jafari <18074432+peimanja@users.noreply.github.com> Date: Fri, 17 Apr 2020 09:20:27 -0700 Subject: [PATCH] Add Metrics for Created and Downloaded artifacts (#35) --- README.md | 7 +- collector/artifacts.go | 176 +++++++++++++++++++++++++++++++++++++++++ collector/collector.go | 28 ++++++- collector/storage.go | 30 ++++--- 4 files changed, 226 insertions(+), 15 deletions(-) create mode 100644 collector/artifacts.go diff --git a/README.md b/README.md index 8c7619c..e812132 100755 --- a/README.md +++ b/README.md @@ -99,7 +99,12 @@ Some metrics are not available with Artifactory OSS license. The exporter return | artifactory_storage_repo_folders | Number of folders in an Artifactory repository. | `name`, `package_type`, `type` | ✅ | | artifactory_storage_repo_files | Number files in an Artifactory repository. | `name`, `package_type`, `type` | ✅ | | artifactory_storage_repo_items | Number Items in an Artifactory repository. | `name`, `package_type`, `type` | ✅ | -| artifactory_storage_repo_percentage | Percentage of space used by an Artifactory repository. | `name`, `package_type`, `type` | ✅ | +| artifactory_artifacts_created_1m | Number of artifacts created in the repo (last 1 minute). | `name`, `package_type`, `type` | ✅ | +| artifactory_artifacts_created_5m | Number of artifacts created in the repo (last 5 minutes). | `name`, `package_type`, `type` | ✅ | +| artifactory_artifacts_created_15m | Number of artifacts created in the repo (last 15 minutes). | `name`, `package_type`, `type` | ✅ | +| artifactory_artifacts_downloaded_1m | Number of artifacts downloaded from the repository (last 1 minute). | `name`, `package_type`, `type` | ✅ | +| artifactory_artifacts_downloaded_5m | Number of artifacts downloaded from the repository (last 5 minutes). | `name`, `package_type`, `type` | ✅ | +| artifactory_artifacts_downloaded_15m | Number of artifacts downloaded from the repository (last 15 minute). | `name`, `package_type`, `type` | ✅ | | artifactory_system_healthy | Is Artifactory working properly (1 = healthy). | | ✅ | | artifactory_system_license | License type and expiry as labels, seconds to expiration as value | `type`, `licensed_to`, `expires` | ✅ | | artifactory_system_version | Version and revision of Artifactory as labels. | `version`, `revision` | ✅ | diff --git a/collector/artifacts.go b/collector/artifacts.go new file mode 100644 index 0000000..7b9a2f6 --- /dev/null +++ b/collector/artifacts.go @@ -0,0 +1,176 @@ +package collector + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + + "github.com/go-kit/kit/log/level" + "github.com/prometheus/client_golang/prometheus" +) + +func (e *Exporter) queryAQL(query []byte) ([]byte, error) { + fullPath := fmt.Sprintf("%s/api/search/aql", e.URI) + tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: !e.sslVerify}} + client := http.Client{ + Timeout: e.timeout, + Transport: tr, + } + level.Debug(e.logger).Log("msg", "Running AQL query", "path", fullPath) + req, err := http.NewRequest("POST", fullPath, bytes.NewBuffer(query)) + req.Header = http.Header{"Content-Type": {"text/plain"}} + if err != nil { + return nil, err + } + + if e.authMethod == "userPass" { + req.SetBasicAuth(e.cred.Username, e.cred.Password) + } else if e.authMethod == "accessToken" { + req.Header.Add("Authorization", "Bearer "+e.cred.AccessToken) + } + + resp, err := client.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if !(resp.StatusCode >= 200 && resp.StatusCode < 300) { + return nil, fmt.Errorf("HTTP status %d", resp.StatusCode) + } + + bodyBytes, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + + return bodyBytes, nil + +} + +type artifact struct { + Repo string `json:"repo,omitempty"` + Name string `json:"name,omitempty"` +} + +type artifactQueryResult struct { + Results []artifact `json:"results,omitempty"` +} + +func (e *Exporter) findArtifacts(period string, queryType string) (artifactQueryResult, error) { + var query string + artifacts := artifactQueryResult{} + level.Debug(e.logger).Log("msg", "Finding all artifacts", "period", period, "queryType", queryType) + switch queryType { + case "created": + query = fmt.Sprintf("items.find({\"modified\" : {\"$last\" : \"%s\"}}).include(\"name\", \"repo\")", period) + case "downloaded": + query = fmt.Sprintf("items.find({\"stat.downloaded\" : {\"$last\" : \"%s\"}}).include(\"name\", \"repo\")", period) + default: + level.Error(e.logger).Log("msg", "Query Type is not supported", "query", queryType) + return artifacts, fmt.Errorf("Query Type is not supported: %s", queryType) + } + resp, err := e.queryAQL([]byte(query)) + if err != nil { + level.Error(e.logger).Log("msg", "There was an error finding artifacts", "queryType", queryType, "period", period, "error", err) + return artifacts, err + } + + if err := json.Unmarshal(resp, &artifacts); err != nil { + level.Debug(e.logger).Log("msg", "There was an issue marshaling AQL response") + e.jsonParseFailures.Inc() + return artifacts, err + } + return artifacts, err +} + +func (e *Exporter) getTotalArtifacts(r []repoSummary) ([]repoSummary, error) { + created1m, err := e.findArtifacts("1minutes", "created") + if err != nil { + return nil, err + } + created5m, err := e.findArtifacts("5minutes", "created") + if err != nil { + return nil, err + } + created15m, err := e.findArtifacts("15minutes", "created") + if err != nil { + return nil, err + } + downloaded1m, err := e.findArtifacts("1minutes", "downloaded") + if err != nil { + return nil, err + } + downloaded5m, err := e.findArtifacts("5minutes", "downloaded") + if err != nil { + return nil, err + } + downloaded15m, err := e.findArtifacts("15minutes", "downloaded") + if err != nil { + return nil, err + } + + repoSummaries := r + for i := range repoSummaries { + for _, k := range created1m.Results { + if repoSummaries[i].Name == k.Repo { + repoSummaries[i].TotalCreate1m++ + } + } + for _, k := range created5m.Results { + if repoSummaries[i].Name == k.Repo { + repoSummaries[i].TotalCreated5m++ + } + } + for _, k := range created15m.Results { + if repoSummaries[i].Name == k.Repo { + repoSummaries[i].TotalCreated15m++ + } + } + for _, k := range downloaded1m.Results { + if repoSummaries[i].Name == k.Repo { + repoSummaries[i].TotalDownloaded1m++ + } + } + for _, k := range downloaded5m.Results { + if repoSummaries[i].Name == k.Repo { + repoSummaries[i].TotalDownloaded5m++ + } + } + for _, k := range downloaded15m.Results { + if repoSummaries[i].Name == k.Repo { + repoSummaries[i].TotalDownloaded15m++ + } + } + } + return repoSummaries, nil +} + +func (e *Exporter) exportArtifacts(repoSummaries []repoSummary, ch chan<- prometheus.Metric) { + for _, repoSummary := range repoSummaries { + for metricName, metric := range artifactsMetrics { + switch metricName { + case "created1m": + level.Debug(e.logger).Log("msg", "Registering metric", "metric", metricName, "repo", repoSummary.Name, "type", repoSummary.Type, "package_type", repoSummary.PackageType, "value", repoSummary.TotalCreate1m) + ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, repoSummary.TotalCreate1m, repoSummary.Name, repoSummary.Type, repoSummary.PackageType) + case "created5m": + level.Debug(e.logger).Log("msg", "Registering metric", "metric", metricName, "repo", repoSummary.Name, "type", repoSummary.Type, "package_type", repoSummary.PackageType, "value", repoSummary.TotalCreated5m) + ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, repoSummary.TotalCreated5m, repoSummary.Name, repoSummary.Type, repoSummary.PackageType) + case "created15m": + level.Debug(e.logger).Log("msg", "Registering metric", "metric", metricName, "repo", repoSummary.Name, "type", repoSummary.Type, "package_type", repoSummary.PackageType, "value", repoSummary.TotalCreated15m) + ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, repoSummary.TotalCreated15m, repoSummary.Name, repoSummary.Type, repoSummary.PackageType) + case "downloaded1m": + level.Debug(e.logger).Log("msg", "Registering metric", "metric", metricName, "repo", repoSummary.Name, "type", repoSummary.Type, "package_type", repoSummary.PackageType, "value", repoSummary.TotalDownloaded1m) + ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, repoSummary.TotalDownloaded1m, repoSummary.Name, repoSummary.Type, repoSummary.PackageType) + case "downloaded5m": + level.Debug(e.logger).Log("msg", "Registering metric", "metric", metricName, "repo", repoSummary.Name, "type", repoSummary.Type, "package_type", repoSummary.PackageType, "value", repoSummary.TotalDownloaded5m) + ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, repoSummary.TotalDownloaded5m, repoSummary.Name, repoSummary.Type, repoSummary.PackageType) + case "downloaded15m": + level.Debug(e.logger).Log("msg", "Registering metric", "metric", metricName, "repo", repoSummary.Name, "type", repoSummary.Type, "package_type", repoSummary.PackageType, "value", repoSummary.TotalDownloaded15m) + ch <- prometheus.MustNewConstMetric(metric, prometheus.GaugeValue, repoSummary.TotalDownloaded15m, repoSummary.Name, repoSummary.Type, repoSummary.PackageType) + } + } + } +} diff --git a/collector/collector.go b/collector/collector.go index f71f594..7ba1b47 100755 --- a/collector/collector.go +++ b/collector/collector.go @@ -62,6 +62,15 @@ var ( "license": newMetric("license", "system", "License type and expiry as labels, seconds to expiration as value", []string{"type", "licensed_to", "expires"}), } + artifactsMetrics = metrics{ + "created1m": newMetric("created_1m", "artifacts", "Number of artifacts created in the repository in the last 1 minute.", repoLabelNames), + "created5m": newMetric("created_5m", "artifacts", "Number of artifacts created in the repository in the last 5 minutes.", repoLabelNames), + "created15m": newMetric("created_15m", "artifacts", "Number of artifacts created in the repository in the last 15 minutes.", repoLabelNames), + "downloaded1m": newMetric("downloaded_1m", "artifacts", "Number of artifacts downloaded from the repository in the last 1 minute.", repoLabelNames), + "downloaded5m": newMetric("downloaded_5m", "artifacts", "Number of artifacts downloaded from the repository in the last 5 minutes.", repoLabelNames), + "downloaded15m": newMetric("downloaded_15m", "artifacts", "Number of artifacts downloaded from the repository in the last 15 minutes.", repoLabelNames), + } + artifactoryUp = newMetric("up", "", "Was the last scrape of Artifactory successful.", nil) ) @@ -123,6 +132,9 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { for _, m := range systemMetrics { ch <- m } + for _, m := range artifactsMetrics { + ch <- m + } ch <- artifactoryUp ch <- e.totalScrapes.Desc() ch <- e.jsonParseFailures.Desc() @@ -174,7 +186,6 @@ func (e *Exporter) fetchHTTP(uri string, path string, cred config.Credentials, a } return bodyBytes, nil - } func (e *Exporter) scrape(ch chan<- prometheus.Metric) (up float64) { @@ -249,7 +260,20 @@ func (e *Exporter) scrape(ch chan<- prometheus.Metric) (up float64) { e.exportFilestore(metricName, metric, storageInfo.FileStoreSummary.FreeSpace, fileStoreType, fileStoreDir, ch) } } - e.extractRepoSummary(storageInfo, ch) + + // Extract repo summaries from storageInfo and register them + repoSummaryList, err := e.extractRepo(storageInfo) + if err != nil { + return 0 + } + e.exportRepo(repoSummaryList, ch) + + // Get Downloaded and Created items for all repo in the last 1 and 5 minutes and add it to repoSummaryList + repoSummaryList, err = e.getTotalArtifacts(repoSummaryList) + if err != nil { + return 0 + } + e.exportArtifacts(repoSummaryList, ch) // Some API endpoints are not available in OSS if licenseType != "oss" { diff --git a/collector/storage.go b/collector/storage.go index 550fb73..5678c04 100644 --- a/collector/storage.go +++ b/collector/storage.go @@ -143,17 +143,23 @@ func (e *Exporter) exportFilestore(metricName string, metric *prometheus.Desc, s } type repoSummary struct { - Name string - Type string - FoldersCount float64 - FilesCount float64 - UsedSpace float64 - ItemsCount float64 - PackageType string - Percentage float64 + Name string + Type string + FoldersCount float64 + FilesCount float64 + UsedSpace float64 + ItemsCount float64 + PackageType string + Percentage float64 + TotalCreate1m float64 + TotalCreated5m float64 + TotalCreated15m float64 + TotalDownloaded1m float64 + TotalDownloaded5m float64 + TotalDownloaded15m float64 } -func (e *Exporter) extractRepoSummary(storageInfo storageInfo, ch chan<- prometheus.Metric) { +func (e *Exporter) extractRepo(storageInfo storageInfo) ([]repoSummary, error) { var err error rs := repoSummary{} repoSummaryList := []repoSummary{} @@ -172,7 +178,7 @@ func (e *Exporter) extractRepoSummary(storageInfo storageInfo, ch chan<- prometh if err != nil { level.Debug(e.logger).Log("msg", "There was an issue parsing repo UsedSpace", "repo", repo.RepoKey, "err", err) e.jsonParseFailures.Inc() - return + return repoSummaryList, err } if repo.Percentage == "N/A" { rs.Percentage = 0 @@ -181,12 +187,12 @@ func (e *Exporter) extractRepoSummary(storageInfo storageInfo, ch chan<- prometh if err != nil { level.Debug(e.logger).Log("msg", "There was an issue parsing repo Percentage", "repo", repo.RepoKey, "err", err) e.jsonParseFailures.Inc() - return + return repoSummaryList, err } } repoSummaryList = append(repoSummaryList, rs) } - e.exportRepo(repoSummaryList, ch) + return repoSummaryList, err } func (e *Exporter) exportRepo(repoSummaries []repoSummary, ch chan<- prometheus.Metric) {