feat: add watcher for events

.github/workflows/build-deploy.yaml

@@ -0,0 +1,80 @@
+name: sniffer build-and deploy
+
+on:
+  pull_request: []
+  release:
+    types: [published]
+  push:
+    branches:
+    - main
+
+jobs:
+  build-sniffer:
+    permissions:
+      packages: write
+    env:
+      container: ghcr.io/converged-computing/sniffer
+    runs-on: ubuntu-latest
+    name: build sniffer
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
+      with:
+        go-version: ^1.21
+
+    - name: Build Containers
+      run: make build-sidecar
+
+    - name: Tag Release Image
+      if: (github.event_name == 'release')
+      run: |
+        tag=${GITHUB_REF#refs/tags/}
+        echo "Tagging and releasing ${{ env.container}}:${tag}"        
+        docker tag ${{ env.container }}:latest ${{ env.container }}:${tag}
+
+    - name: GHCR Login
+      if: (github.event_name != 'pull_request')
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Deploy Container
+      if: (github.event_name != 'pull_request')
+      run: docker push ${{ env.container }} --all-tags
+
+  build-scheduler-sniffer:
+    permissions:
+      packages: write
+    env:
+      container: ghcr.io/converged-computing/scheduler-sniffer
+    runs-on: ubuntu-latest
+    name: build scheduler-sniffer
+    steps:
+    - uses: actions/checkout@v4
+    - uses: actions/setup-go@v4
+      with:
+        go-version: ^1.21
+
+    - name: Build Containers
+      run: make build
+
+    - name: Tag Release Image
+      if: (github.event_name == 'release')
+      run: |
+        tag=${GITHUB_REF#refs/tags/}
+        echo "Tagging and releasing ${{ env.container}}:${tag}"        
+        docker tag ${{ env.container }}:latest ${{ env.container }}:${tag}
+
+    - name: GHCR Login
+      if: (github.event_name != 'pull_request')
+      uses: docker/login-action@v2
+      with:
+        registry: ghcr.io
+        username: ${{ github.actor }}
+        password: ${{ secrets.GITHUB_TOKEN }}
+
+    - name: Deploy Container
+      if: (github.event_name != 'pull_request')
+      run: docker push ${{ env.container }} --all-tags

Makefile

@@ -17,7 +17,7 @@ ARCH ?= "amd64"
 # These are passed to build the sidecar
 REGISTRY ?= ghcr.io/converged-computing
 SCHEDULER_IMAGE ?= scheduler-sniffer
-SIDECAR_IMAGE ?= sniffer-sidecar:latest
+SIDECAR_IMAGE ?= sniffer:latest
 
 .PHONY: all build clone update
 all: prepare build-sidecar build
@@ -35,7 +35,7 @@ prepare: clone clone-k8s
 	# This ensures the sig-scheduler image has the same grpc
 	cp -R sniffer/api $(UPSTREAM_K8S)/pkg/sniffer
 
-    # These basically allow us to wrap the default scheduler so we can run the command (and trace it)
+	# These basically allow us to wrap the default scheduler so we can run the command (and trace it)
 	cp scheduler/* $(UPSTREAM_K8S)/pkg/scheduler/
 	cp src/cmd/scheduler/main.go $(UPSTREAM)/cmd/scheduler/main.go
 	cp src/manifests/install/charts/as-a-second-scheduler/templates/*.yaml $(UPSTREAM)/manifests/install/charts/as-a-second-scheduler/templates/

README.md

@@ -38,13 +38,24 @@ kubectl get events -o wide |  awk {'print $4" " $5" " $6'} | column -t | grep sn
 You can also look at the sniffer logs to see the binding events printed:
 
 ```bash
-$ kubectl exec sniffer-57f85c77d8-lgx9j -- cat /tmp/sniffer.log
-Defaulted container "sniffer" out of: sniffer, scheduler
-  DEBUG:  {"pod":"job-z2m8m","endpoint":"schedulePod","node":"kind-control-plane","event":"ScheduleSuccess","timestamp":"2024-04-21 20:54:05.171920853 +0000 UTC m=+78.753596979"}
-  DEBUG:  {"pod":"job-z2m8m","endpoint":"bindingCycle","node":"kind-control-plane","event":"BindingSuccess","timestamp":"2024-04-21 20:54:05.184822919 +0000 UTC m=+78.766499043"}
+$ kubectl exec -it sniffer-74c76ff4f8-hrqdn -c watcher -- cat /tmp/logs/sniffer.log
+```
+```console
+{"object":"Pod","name":"local-path-provisioner-7577fdbbfb-w84m2","endpoint":"podUpdate","event":"ContainersReady","timestamp":"2024-04-20 18:22:03 +0000 UTC"}
+{"object":"Pod","name":"local-path-provisioner-7577fdbbfb-w84m2","endpoint":"podUpdate","event":"PodScheduled","timestamp":"2024-04-20 18:22:00 +0000 UTC"}
+```
+
+You can grep for "Node" or "Pod" to filter events:
+
+```bash
+$ kubectl exec -it sniffer-74c76ff4f8-hrqdn -c watcher -- cat /tmp/logs/sniffer.log | grep Node
+```
+```console
+{"object":"Node","name":"kind-control-plane","endpoint":"nodeUpdate","reason":"KubeletReady","message":"kubelet is posting ready status","event":"Ready","timestamp":"2024-04-22 01:08:24 +0000 UTC"}
+{"object":"Node","name":"kind-control-plane","endpoint":"nodeUpdate","extra":{"capacity-cpu":"12","capacity-ephemeral-storage":"1921208544Ki","capacity-hugepages-1Gi":"0","capacity-hugepages-2Mi":"0","capacity-memory":"32512128Ki","capacity-pods":"110","allocatable-hugepages-2Mi":"0","allocatable-memory":"32512128Ki","allocatable-pods":"110","allocatable-cpu":"12","allocatable-ephemeral-storage":"1921208544Ki","allocatable-hugepages-1Gi":"0"}}
 ```
 
-This is still early in development, more soon.
+This is still early in development, more soon. I think likely we will need to update the output file to an actual database.
 
 ### Build Logic
 

hack/kind-build.sh

@@ -16,13 +16,13 @@ make
 
 # We load into kind so we don't need to push/pull and use up internet data ;)
 kind load docker-image ${REGISTRY}/scheduler-sniffer:latest
-kind load docker-image ${REGISTRY}/sniffer-sidecar:latest
+kind load docker-image ${REGISTRY}/sniffer:latest
 
 # And then install using the charts. The pull policy ensures we use the loaded ones
 cd ${ROOT}/upstreams/sig-scheduler-plugins/manifests/install/charts
 helm uninstall sniffer || true
 helm install \
   --set sniffer.pullPolicy=Never \
-  --set sniffer.image=${REGISTRY}/sniffer-sidecar:latest \
+  --set sniffer.image=${REGISTRY}/sniffer:latest \
   --set scheduler.image=${REGISTRY}/scheduler-sniffer:latest \
   --set scheduler.pullPolicy=Never sniffer as-a-second-scheduler/

sniffer/Makefile

@@ -18,8 +18,15 @@ build:
 	docker build -f Dockerfile --build-arg ARCH="amd64" --build-arg RELEASE_VERSION="$(RELEASE_VERSION)" -t $(LOCAL_REGISTRY)/$(LOCAL_IMAGE) .
 
 .PHONY: sniffer
-sniffer: 
-	go build -ldflags '-w' -o bin/sniffer cmd/main.go
+sniffer: service watcher
+
+.PHONY: service
+service: 
+	go build -ldflags '-w' -o bin/sniffer cmd/service/service.go
+
+.PHONY: watcher
+watcher: 
+	go build -ldflags '-w' -o bin/watcher cmd/watcher/watcher.go
 
 .PHONY: protoc
 protoc: $(LOCALBIN)

sniffer/cmd/main.go → sniffer/cmd/service/service.go

@@ -16,21 +16,20 @@ import (
 )
 
 const (
-	defaultPort    = ":4242"
-	defaultLogfile = "/tmp/sniffer.log"
+	defaultPort = ":4242"
 )
 
 var responsechan chan string
 
 func main() {
 	grpcPort := flag.String("port", defaultPort, "Port for grpc service")
-
+	logfile := flag.String("logfile", logger.DefaultLogFile, "Default log file to write to")
 	flag.Parse()
 
 	// Sniffer logger to file and to terminal
 	// This can eventually be a flag, but just going to set for now
 	// It shall be a very chonky file. Oh lawd he comin!
-	l := logger.NewDebugLogger(logger.LevelDebug, "/tmp/sniffer.log")
+	l := logger.NewDebugLogger(logger.LevelDebug, *logfile)
 
 	// Ensure our port starts with :
 	port := *grpcPort

sniffer/cmd/watcher/watcher.go

@@ -0,0 +1,17 @@
+package main
+
+import (
+	"flag"
+
+	"github.com/converged-computing/scheduler-sniffer/sniffer/pkg/logger"
+	"github.com/converged-computing/scheduler-sniffer/sniffer/pkg/watcher"
+)
+
+func main() {
+	logfile := flag.String("logfile", logger.DefaultLogFile, "Default log file to write to")
+	flag.Parse()
+
+	// Create a new watcher with the logfile
+	w := watcher.NewWatcher(*logfile)
+	w.Run()
+}

sniffer/pkg/logger/logger.go

@@ -19,6 +19,12 @@ const (
 	LevelDebug
 )
 
+// This should correspond to a shared empty dir if being
+// used alongside the scheduler-sniffer
+var (
+	DefaultLogFile = "/tmp/logs/sniffer.log"
+)
+
 type DebugLogger struct {
 	level    int
 	Filename string
@@ -49,20 +55,21 @@ func (l *DebugLogger) Stop() error {
 }
 
 // Logging functions you should use!
+// Note that prefixes are disabled because we want to parse json in each line
 func (l *DebugLogger) Info(message ...any) error {
-	return l.log(LevelInfo, "   INFO: ", message...)
+	return l.log(LevelInfo, "", message...)
 }
 func (l *DebugLogger) Error(message ...any) error {
-	return l.log(LevelError, "  ERROR: ", message...)
+	return l.log(LevelError, "", message...)
 }
 func (l *DebugLogger) Debug(message ...any) error {
-	return l.log(LevelDebug, "  DEBUG: ", message...)
+	return l.log(LevelDebug, "", message...)
 }
 func (l *DebugLogger) Verbose(message ...any) error {
-	return l.log(LevelVerbose, "VERBOSE: ", message...)
+	return l.log(LevelVerbose, "", message...)
 }
 func (l *DebugLogger) Warning(message ...any) error {
-	return l.log(LevelWarning, "WARNING: ", message...)
+	return l.log(LevelWarning, "", message...)
 }
 
 // log is the shared class function for actually printing to the log

sniffer/pkg/service/service.go

@@ -1,43 +1,24 @@
 package service
 
 import (
-	"encoding/json"
 	"fmt"
 
 	"context"
 
 	pb "github.com/converged-computing/scheduler-sniffer/sniffer/api"
 	"github.com/converged-computing/scheduler-sniffer/sniffer/pkg/logger"
+	"github.com/converged-computing/scheduler-sniffer/sniffer/pkg/types"
 )
 
 type Sniffer struct {
 	pb.UnimplementedSnifferServiceServer
 	Logger *logger.DebugLogger
 }
 
-// A SnifferDatum holds one entry of data
-type SnifferDatum struct {
-	Pod       string `json:"pod"`
-	Endpoint  string `json:"endpoint"`
-	Node      string `json:"node"`
-	Event     string `json:"event"`
-	Timestamp string `json:"timestamp"`
-}
-
-// ToJson serializes to json
-func (d *SnifferDatum) ToJson() (string, error) {
-	out, err := json.Marshal(d)
-	if err != nil {
-		fmt.Printf("error marshalling: %s\n", err)
-		return "", err
-	}
-	return string(out), nil
-}
-
 // Send receives data from Kubernetes->scheduler module, specifically when a bind is successful
 // We are currently not using the in.Payload
 func (s *Sniffer) Send(ctx context.Context, in *pb.SendRequest) (*pb.SendResponse, error) {
-	datum := SnifferDatum{Pod: in.Pod, Endpoint: in.Endpoint, Node: in.Node, Event: in.Event, Timestamp: in.Timestamp}
+	datum := types.SnifferDatum{Name: in.Pod, Object: "Pod", Endpoint: in.Endpoint, Node: in.Node, Event: in.Event, Timestamp: in.Timestamp}
 	out, err := datum.ToJson()
 	if err == nil {
 		s.Logger.Debug("%s", out)

sniffer/pkg/types/types.go

@@ -0,0 +1,29 @@
+package types
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// A SnifferDatum holds one entry of data
+type SnifferDatum struct {
+	Object    string          `json:"object"`
+	Name      string          `json:"name"`
+	Endpoint  string          `json:"endpoint,omitempty"`
+	Reason    string          `json:"reason,omitempty"`
+	Message   string          `json:"message,omitempty"`
+	Node      string          `json:"node,omitempty"`
+	Event     string          `json:"event,omitempty"`
+	Extra     json.RawMessage `json:"extra,omitempty"`
+	Timestamp string          `json:"timestamp,omitempty"`
+}
+
+// ToJson serializes to json
+func (d *SnifferDatum) ToJson() (string, error) {
+	out, err := json.Marshal(&d)
+	if err != nil {
+		fmt.Printf("error marshalling: %s\n", err)
+		return "", err
+	}
+	return string(out), nil
+}