From 357fd689e58f56cc3626799c33dada210b7ff58a Mon Sep 17 00:00:00 2001 From: Darren Shepherd Date: Thu, 28 Oct 2021 09:16:07 -0700 Subject: [PATCH] Add TPM support --- .drone.yml | 43 ---------- cmd/rancherd/gettpmhash/gettpmhash.go | 28 +++++++ cmd/rancherd/main.go | 2 + pkg/cacerts/cacerts.go | 35 +++++++- pkg/config/runtime.go | 5 +- pkg/discovery/discovery.go | 14 +++- pkg/plan/bootstrap.go | 23 ++---- pkg/probe/probe.go | 16 ++-- pkg/rancher/run.go | 11 ++- pkg/tpm/get.go | 113 ++++++++++++++++++++++++++ pkg/tpm/tpm.go | 106 ++++++++++++++++++++++++ pkg/tpm/tpm_attestor.go | 80 ++++++++++++++++++ 12 files changed, 395 insertions(+), 81 deletions(-) create mode 100644 cmd/rancherd/gettpmhash/gettpmhash.go create mode 100644 pkg/tpm/get.go create mode 100644 pkg/tpm/tpm.go create mode 100644 pkg/tpm/tpm_attestor.go diff --git a/.drone.yml b/.drone.yml index 23a50af..b338563 100644 --- a/.drone.yml +++ b/.drone.yml @@ -83,46 +83,3 @@ volumes: - name: docker host: path: /var/run/docker.sock - ---- -kind: pipeline -name: arm - -platform: - os: linux - arch: arm - -steps: -- name: build - image: rancher/dapper:v0.4.1 - commands: - - dapper ci - volumes: - - name: docker - path: /var/run/docker.sock - -- name: github_binary_release - image: plugins/github-release - settings: - api_key: - from_secret: github_token - prerelease: true - checksum: - - sha256 - checksum_file: CHECKSUMsum-arm.txt - checksum_flatten: true - files: - - "dist/artifacts/*" - when: - instance: - - drone-publish.rancher.io - ref: - - refs/head/master - - refs/tags/* - event: - - tag - -volumes: -- name: docker - host: - path: /var/run/docker.sock diff --git a/cmd/rancherd/gettpmhash/gettpmhash.go b/cmd/rancherd/gettpmhash/gettpmhash.go new file mode 100644 index 0000000..249b762 --- /dev/null +++ b/cmd/rancherd/gettpmhash/gettpmhash.go @@ -0,0 +1,28 @@ +package gettpmhash + +import ( + "fmt" + + "github.com/rancher/rancherd/pkg/tpm" + cli "github.com/rancher/wrangler-cli" + "github.com/spf13/cobra" +) + +func NewGetTPMHash() *cobra.Command { + return cli.Command(&GetTPMHash{}, cobra.Command{ + Use: "get-tpm-hash", + Short: "Print TPM hash to identify this machine", + }) +} + +type GetTPMHash struct { +} + +func (p *GetTPMHash) Run(cmd *cobra.Command, args []string) error { + str, err := tpm.GetPubHash() + if err != nil { + return err + } + fmt.Println(str) + return nil +} diff --git a/cmd/rancherd/main.go b/cmd/rancherd/main.go index 5a7d6c1..47bc785 100644 --- a/cmd/rancherd/main.go +++ b/cmd/rancherd/main.go @@ -3,6 +3,7 @@ package main import ( "github.com/rancher/rancherd/cmd/rancherd/bootstrap" "github.com/rancher/rancherd/cmd/rancherd/gettoken" + "github.com/rancher/rancherd/cmd/rancherd/gettpmhash" "github.com/rancher/rancherd/cmd/rancherd/info" "github.com/rancher/rancherd/cmd/rancherd/probe" "github.com/rancher/rancherd/cmd/rancherd/resetadmin" @@ -31,6 +32,7 @@ func main() { retry.NewRetry(), upgrade.NewUpgrade(), info.NewInfo(), + gettpmhash.NewGetTPMHash(), ) cli.Main(root) } diff --git a/pkg/cacerts/cacerts.go b/pkg/cacerts/cacerts.go index a3f3245..c9cd74d 100644 --- a/pkg/cacerts/cacerts.go +++ b/pkg/cacerts/cacerts.go @@ -14,6 +14,7 @@ import ( url2 "net/url" "time" + "github.com/rancher/rancherd/pkg/tpm" "github.com/rancher/wrangler/pkg/randomtoken" ) @@ -42,18 +43,33 @@ func get(server, token, path string, clusterToken bool) ([]byte, string, error) } u.Path = path - req, err := http.NewRequest(http.MethodGet, u.String(), nil) + var ( + isTPM bool + ) + if !clusterToken { + isTPM, token, err = tpm.ResolveToken(token) + if err != nil { + return nil, "", err + } + } + + cacert, caChecksum, err := CACerts(server, token, clusterToken) if err != nil { return nil, "", err } - if !clusterToken { - req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString([]byte(token))) + + if isTPM { + data, err := tpm.Get(cacert, u.String(), nil) + return data, caChecksum, err } - cacert, caChecksum, err := CACerts(server, token, clusterToken) + req, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return nil, "", err } + if !clusterToken { + req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString([]byte(token))) + } var resp *http.Response if len(cacert) == 0 { @@ -103,6 +119,13 @@ func CACerts(server, token string, clusterToken bool) ([]byte, string, error) { if !clusterToken { requestURL = fmt.Sprintf("https://%s/v1-rancheros/cacerts", url.Host) } + + if resp, err := http.Get(requestURL); err == nil { + _, _ = ioutil.ReadAll(resp.Body) + resp.Body.Close() + return nil, "", nil + } + req, err := http.NewRequest(http.MethodGet, requestURL, nil) if err != nil { return nil, "", err @@ -121,6 +144,10 @@ func CACerts(server, token string, clusterToken bool) ([]byte, string, error) { return nil, "", err } + if resp.StatusCode != http.StatusOK { + return nil, "", fmt.Errorf("response %d: %s getting cacerts: %s", resp.StatusCode, resp.Status, data) + } + if resp.Header.Get("X-Cattle-Hash") != hash(token, nonce, data) { return nil, "", fmt.Errorf("response hash (%s) does not match (%s)", resp.Header.Get("X-Cattle-Hash"), diff --git a/pkg/config/runtime.go b/pkg/config/runtime.go index 489b4ed..0038088 100644 --- a/pkg/config/runtime.go +++ b/pkg/config/runtime.go @@ -3,8 +3,9 @@ package config import "strings" var ( - RuntimeRKE2 Runtime = "rke2" - RuntimeK3S Runtime = "k3s" + RuntimeRKE2 Runtime = "rke2" + RuntimeK3S Runtime = "k3s" + RuntimeUnknown Runtime = "unknown" ) type Runtime string diff --git a/pkg/discovery/discovery.go b/pkg/discovery/discovery.go index 27df2a8..d8dd826 100644 --- a/pkg/discovery/discovery.go +++ b/pkg/discovery/discovery.go @@ -106,6 +106,9 @@ func discoverServerAndRole(ctx context.Context, cfg *config.Config) (string, boo } func (j *joinServer) addresses(params map[string]string, discovery *discover.Discover) ([]string, error) { + if params["provider"] == "mdns" { + params["v6"] = "false" + } addrs, err := discovery.Addrs(discover.Config(params).String(), log.Default()) if err != nil { return nil, err @@ -146,7 +149,7 @@ func (j *joinServer) loop(ctx context.Context, count int, params map[string]stri } resp, err := insecureHTTPClient.Do(req) if err != nil { - logrus.Errorf("failed to connect to %s: %v", url, err) + logrus.Infof("failed to connect to %s: %v", url, err) allAgree = false continue } @@ -154,7 +157,7 @@ func (j *joinServer) loop(ctx context.Context, count int, params map[string]stri data, err := ioutil.ReadAll(resp.Body) resp.Body.Close() if err != nil || resp.StatusCode != http.StatusOK { - logrus.Errorf("failed to read response from %s: code %d: %v", url, resp.StatusCode, err) + logrus.Infof("failed to read response from %s: code %d: %v", url, resp.StatusCode, err) allAgree = false continue } @@ -181,6 +184,11 @@ func (j *joinServer) loop(ctx context.Context, count int, params map[string]stri } } + if len(addrs) == 0 { + logrus.Infof("No available peers") + return "", false + } + if firstID != j.id { logrus.Infof("Waiting for peer %s from %v to initialize", addrs[0], addrs) return "", false @@ -219,7 +227,7 @@ func newJoinServer(ctx context.Context, cacheDuration string, port int64) (*join } if cacheDuration == "" { - cacheDuration = "5m" + cacheDuration = "1m" } duration, err := time.ParseDuration(cacheDuration) diff --git a/pkg/plan/bootstrap.go b/pkg/plan/bootstrap.go index 43740f7..6470454 100644 --- a/pkg/plan/bootstrap.go +++ b/pkg/plan/bootstrap.go @@ -49,28 +49,19 @@ func toJoinPlan(cfg *config.Config, dataDir string) (*applyinator.Plan, error) { } plan := plan{} - k8sVersion, err := versions.K8sVersion(cfg.KubernetesVersion) - if err != nil { - return nil, err - } - if err := plan.addFile(join.ToScriptFile(cfg, dataDir)); err != nil { return nil, err } - if err := plan.addFile(runtime.ToFile(&cfg.RuntimeConfig, config.GetRuntime(k8sVersion), false)); err != nil { - return nil, err - } if err := plan.addInstruction(join.ToInstruction(cfg, dataDir)); err != nil { return nil, err } - if err := plan.addInstruction(probe.ToInstruction(cfg.RuntimeInstallerImage, cfg.SystemDefaultRegistry, k8sVersion)); err != nil { + if err := plan.addInstruction(probe.ToInstruction()); err != nil { return nil, err } - if err := plan.addProbesForRoles(cfg); err != nil { + if err := plan.addProbesForJoin(cfg); err != nil { return nil, err } - plan.addPrePostInstructions(cfg, "") return (*applyinator.Plan)(&plan), nil } @@ -95,7 +86,7 @@ func (p *plan) addInstructions(cfg *config.Config, dataDir string) error { return err } - if err := p.addInstruction(probe.ToInstruction(cfg.RuntimeInstallerImage, cfg.SystemDefaultRegistry, k8sVersion)); err != nil { + if err := p.addInstruction(probe.ToInstruction()); err != nil { return err } @@ -204,12 +195,8 @@ func (p *plan) addFile(file *applyinator.File, err error) error { return nil } -func (p *plan) addProbesForRoles(cfg *config.Config) error { - k8sVersion, err := versions.K8sVersion(cfg.KubernetesVersion) - if err != nil { - return err - } - p.Probes = probe.ProbesForRole(&cfg.RuntimeConfig, config.GetRuntime(k8sVersion)) +func (p *plan) addProbesForJoin(cfg *config.Config) error { + p.Probes = probe.ProbesForJoin(&cfg.RuntimeConfig) return nil } diff --git a/pkg/probe/probe.go b/pkg/probe/probe.go index a0259d8..d93d5c1 100644 --- a/pkg/probe/probe.go +++ b/pkg/probe/probe.go @@ -61,13 +61,13 @@ func replaceRuntime(str string, runtime config.Runtime) string { return fmt.Sprintf(str, runtime) } -func ProbesForRole(config *config.RuntimeConfig, runtime config.Runtime) map[string]prober.Probe { - if roles.IsControlPlane(config.Role) { - return AllProbes(runtime) +func ProbesForJoin(cfg *config.RuntimeConfig) map[string]prober.Probe { + if roles.IsControlPlane(cfg.Role) { + return AllProbes(config.RuntimeUnknown) } return replaceRuntimeForProbes(map[string]prober.Probe{ "kubelet": probes["kubelet"], - }, runtime) + }, config.RuntimeUnknown) } func AllProbes(runtime config.Runtime) map[string]prober.Probe { @@ -77,6 +77,12 @@ func AllProbes(runtime config.Runtime) map[string]prober.Probe { func replaceRuntimeForProbes(probes map[string]prober.Probe, runtime config.Runtime) map[string]prober.Probe { result := map[string]prober.Probe{} for k, v := range probes { + // we don't know the runtime to find the file + if runtime == config.RuntimeUnknown && (v.HTTPGetAction.CACert+ + v.HTTPGetAction.ClientCert+ + v.HTTPGetAction.ClientKey) != "" { + continue + } v.HTTPGetAction.CACert = replaceRuntime(v.HTTPGetAction.CACert, runtime) v.HTTPGetAction.ClientCert = replaceRuntime(v.HTTPGetAction.ClientCert, runtime) v.HTTPGetAction.ClientKey = replaceRuntime(v.HTTPGetAction.ClientKey, runtime) @@ -85,7 +91,7 @@ func replaceRuntimeForProbes(probes map[string]prober.Probe, runtime config.Runt return result } -func ToInstruction(imageOverride string, systemDefaultRegistry string, k8sVersion string) (*applyinator.Instruction, error) { +func ToInstruction() (*applyinator.Instruction, error) { cmd, err := self.Self() if err != nil { return nil, fmt.Errorf("resolving location of %s: %w", os.Args[0], err) diff --git a/pkg/rancher/run.go b/pkg/rancher/run.go index fbc5045..3698daa 100644 --- a/pkg/rancher/run.go +++ b/pkg/rancher/run.go @@ -16,12 +16,11 @@ var defaultValues = map[string]interface{}{ "ingress": map[string]interface{}{ "enabled": false, }, - "features": "multi-cluster-management=false", - "antiAffinity": "required", - "replicas": -3, - "tls": "external", - "hostPort": 8443, - "noDefaultAdmin": true, + "features": "multi-cluster-management=false", + "antiAffinity": "required", + "replicas": -3, + "tls": "external", + "hostPort": 8443, } func GetRancherValues(dataDir string) string { diff --git a/pkg/tpm/get.go b/pkg/tpm/get.go new file mode 100644 index 0000000..f39a268 --- /dev/null +++ b/pkg/tpm/get.go @@ -0,0 +1,113 @@ +package tpm + +import ( + "crypto/tls" + "crypto/x509" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "strings" + "time" + + "github.com/google/go-attestation/attest" + "github.com/gorilla/websocket" + "github.com/sirupsen/logrus" +) + +func Get(cacerts []byte, url string, header http.Header) ([]byte, error) { + dialer := websocket.DefaultDialer + if len(cacerts) > 0 { + pool := x509.NewCertPool() + pool.AppendCertsFromPEM(cacerts) + dialer = &websocket.Dialer{ + Proxy: http.ProxyFromEnvironment, + HandshakeTimeout: 45 * time.Second, + TLSClientConfig: &tls.Config{ + RootCAs: pool, + }, + } + } + + attestationData, aikBytes, err := getAttestationData() + if err != nil { + return nil, err + } + + token, err := getToken(attestationData) + if err != nil { + return nil, err + } + + if header == nil { + header = http.Header{} + } + header.Add("Authorization", token) + wsURL := strings.Replace(url, "http", "ws", 1) + logrus.Infof("Dialing %s with Authorization: %s", wsURL, token) + conn, _, err := dialer.Dial(wsURL, header) + if err != nil { + return nil, err + } + defer conn.Close() + + _, msg, err := conn.NextReader() + if err != nil { + return nil, fmt.Errorf("reading challenge: %w", err) + } + + var challenge Challenge + if err := json.NewDecoder(msg).Decode(&challenge); err != nil { + return nil, fmt.Errorf("unmarshaling Challenge: %w", err) + } + + resp, err := getChallengeResponse(challenge.EC, aikBytes) + if err != nil { + return nil, err + } + + writer, err := conn.NextWriter(websocket.BinaryMessage) + if err != nil { + return nil, err + } + defer writer.Close() + + if err := json.NewEncoder(writer).Encode(resp); err != nil { + return nil, fmt.Errorf("encoding ChallengeResponse: %w", err) + } + + if err := writer.Close(); err != nil { + return nil, fmt.Errorf("closing websocket writer: %w", err) + } + + _, msg, err = conn.NextReader() + if err != nil { + return nil, fmt.Errorf("reading payload from tpm get: %w", err) + } + + return ioutil.ReadAll(msg) +} + +func getChallengeResponse(ec *attest.EncryptedCredential, aikBytes []byte) (*ChallengeResponse, error) { + tpm, err := attest.OpenTPM(&attest.OpenConfig{ + TPMVersion: attest.TPMVersion20, + }) + if err != nil { + return nil, fmt.Errorf("opening tpm: %w", err) + } + defer tpm.Close() + + aik, err := tpm.LoadAK(aikBytes) + if err != nil { + return nil, err + } + defer aik.Close(tpm) + + secret, err := aik.ActivateCredential(tpm, *ec) + if err != nil { + return nil, fmt.Errorf("failed to activate credential: %w", err) + } + return &ChallengeResponse{ + Secret: secret, + }, nil +} diff --git a/pkg/tpm/tpm.go b/pkg/tpm/tpm.go new file mode 100644 index 0000000..466744a --- /dev/null +++ b/pkg/tpm/tpm.go @@ -0,0 +1,106 @@ +package tpm + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + "github.com/google/go-attestation/attest" +) + +func ResolveToken(token string) (bool, string, error) { + if !strings.HasPrefix(token, "tpm://") { + return false, token, nil + } + + hash, err := GetPubHash() + return true, hash, err +} + +func GetPubHash() (string, error) { + ek, err := getEK() + if err != nil { + return "", fmt.Errorf("getting EK: %w", err) + } + + hash, err := getPubHash(ek) + if err != nil { + return "", fmt.Errorf("hashing EK: %w", err) + } + + return hash, nil +} + +func getEK() (*attest.EK, error) { + var err error + tpm, err := attest.OpenTPM(&attest.OpenConfig{ + TPMVersion: attest.TPMVersion20, + }) + if err != nil { + return nil, fmt.Errorf("opening tpm: %w", err) + } + defer tpm.Close() + + eks, err := tpm.EKs() + if err != nil { + return nil, err + } + + if len(eks) == 0 { + return nil, fmt.Errorf("failed to find EK") + } + + return &eks[0], nil +} + +func getToken(data *AttestationData) (string, error) { + bytes, err := json.Marshal(data) + if err != nil { + return "", fmt.Errorf("marshalling attestation data: %w", err) + } + + return "Bearer TPM" + base64.StdEncoding.EncodeToString(bytes), nil +} + +func getAttestationData() (*AttestationData, []byte, error) { + var err error + tpm, err := attest.OpenTPM(&attest.OpenConfig{ + TPMVersion: attest.TPMVersion20, + }) + if err != nil { + return nil, nil, fmt.Errorf("opening tpm: %w", err) + } + defer tpm.Close() + + eks, err := tpm.EKs() + if err != nil { + return nil, nil, err + } + ak, err := tpm.NewAK(nil) + if err != nil { + return nil, nil, err + } + defer ak.Close(tpm) + params := ak.AttestationParameters() + + if len(eks) == 0 { + return nil, nil, fmt.Errorf("failed to find EK") + } + + ek := &eks[0] + ekBytes, err := EncodeEK(ek) + if err != nil { + return nil, nil, err + } + + aikBytes, err := ak.Marshal() + if err != nil { + return nil, nil, fmt.Errorf("marshaling AK: %w", err) + } + + return &AttestationData{ + EK: ekBytes, + AK: ¶ms, + }, aikBytes, nil +} diff --git a/pkg/tpm/tpm_attestor.go b/pkg/tpm/tpm_attestor.go new file mode 100644 index 0000000..632241a --- /dev/null +++ b/pkg/tpm/tpm_attestor.go @@ -0,0 +1,80 @@ +/* + ** Copyright 2019 Bloomberg Finance L.P. + ** + ** Licensed under the Apache License, Version 2.0 (the "License"); + ** you may not use this file except in compliance with the License. + ** You may obtain a copy of the License at + ** + ** http://www.apache.org/licenses/LICENSE-2.0 + ** + ** Unless required by applicable law or agreed to in writing, software + ** distributed under the License is distributed on an "AS IS" BASIS, + ** WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ** See the License for the specific language governing permissions and + ** limitations under the License. + */ + +package tpm + +import ( + "crypto/sha256" + "encoding/pem" + "fmt" + + "github.com/google/certificate-transparency-go/x509" + "github.com/google/go-attestation/attest" +) + +type AttestationData struct { + EK []byte + AK *attest.AttestationParameters +} + +type Challenge struct { + EC *attest.EncryptedCredential +} + +type KeyData struct { + Keys []string `json:"keys"` +} + +type ChallengeResponse struct { + Secret []byte +} + +func getPubHash(ek *attest.EK) (string, error) { + data, err := pubBytes(ek) + if err != nil { + return "", err + } + pubHash := sha256.Sum256(data) + hashEncoded := fmt.Sprintf("%x", pubHash) + return hashEncoded, nil +} + +func EncodeEK(ek *attest.EK) ([]byte, error) { + if ek.Certificate != nil { + return pem.EncodeToMemory(&pem.Block{ + Type: "CERTIFICATE", + Bytes: ek.Certificate.Raw, + }), nil + } + + data, err := pubBytes(ek) + if err != nil { + return nil, err + } + + return pem.EncodeToMemory(&pem.Block{ + Type: "PUBLIC KEY", + Bytes: data, + }), nil +} + +func pubBytes(ek *attest.EK) ([]byte, error) { + data, err := x509.MarshalPKIXPublicKey(ek.Public) + if err != nil { + return nil, fmt.Errorf("error marshaling ec public key: %v", err) + } + return data, nil +}