diff --git a/pkg/cwhub/cwhub_test.go b/pkg/cwhub/cwhub_test.go index c418820b91e..f26a437e0f9 100644 --- a/pkg/cwhub/cwhub_test.go +++ b/pkg/cwhub/cwhub_test.go @@ -29,10 +29,9 @@ const mockURLTemplate = "https://cdn-hub.crowdsec.net/crowdsecurity/%s/%s" var responseByPath map[string]string -// testHub initializes a temporary hub with an empty json file, optionally updating it. -func testHub(t *testing.T, update bool) *Hub { - tmpDir, err := os.MkdirTemp("", "testhub") - require.NoError(t, err) +// testHubOld initializes a temporary hub with an empty json file, optionally updating it. +func testHubOld(t *testing.T, update bool) *Hub { + tmpDir := t.TempDir() local := &csconfig.LocalHubCfg{ HubDir: filepath.Join(tmpDir, "crowdsec", "hub"), @@ -41,7 +40,7 @@ func testHub(t *testing.T, update bool) *Hub { InstallDataDir: filepath.Join(tmpDir, "installed-data"), } - err = os.MkdirAll(local.HubDir, 0o700) + err := os.MkdirAll(local.HubDir, 0o700) require.NoError(t, err) err = os.MkdirAll(local.InstallDir, 0o700) @@ -53,10 +52,6 @@ func testHub(t *testing.T, update bool) *Hub { err = os.WriteFile(local.HubIndexFile, []byte("{}"), 0o644) require.NoError(t, err) - t.Cleanup(func() { - os.RemoveAll(tmpDir) - }) - hub, err := NewHub(local, log.StandardLogger()) require.NoError(t, err) @@ -91,7 +86,7 @@ func envSetup(t *testing.T) *Hub { // Mock the http client HubClient.Transport = newMockTransport() - hub := testHub(t, true) + hub := testHubOld(t, true) return hub } diff --git a/pkg/cwhub/hub.go b/pkg/cwhub/hub.go index d54569c077c..998a4032359 100644 --- a/pkg/cwhub/hub.go +++ b/pkg/cwhub/hub.go @@ -36,7 +36,7 @@ func (h *Hub) GetDataDir() string { // and check for unmanaged items. func NewHub(local *csconfig.LocalHubCfg, logger *logrus.Logger) (*Hub, error) { if local == nil { - return nil, errors.New("no hub configuration found") + return nil, errors.New("no hub configuration provided") } if logger == nil { @@ -149,12 +149,14 @@ func (h *Hub) ItemStats() []string { return ret } +var ErrUpdateAfterSync = errors.New("cannot update hub index after load/sync") + // Update downloads the latest version of the index and writes it to disk if it changed. -// It cannot be called after Load() unless the hub is completely empty. +// It cannot be called after Load() unless the index was completely empty. func (h *Hub) Update(ctx context.Context, indexProvider IndexProvider, withContent bool) error { - if len(h.pathIndex) > 0 { + if len(h.items) > 0 { // if this happens, it's a bug. - return errors.New("cannot update hub after items have been loaded") + return ErrUpdateAfterSync } downloaded, err := indexProvider.FetchIndex(ctx, h.local.HubIndexFile, withContent, h.logger) diff --git a/pkg/cwhub/hub_test.go b/pkg/cwhub/hub_test.go index 6763d26485b..8707bf0d2e1 100644 --- a/pkg/cwhub/hub_test.go +++ b/pkg/cwhub/hub_test.go @@ -2,87 +2,262 @@ package cwhub import ( "context" - "fmt" + "net/http" + "net/http/httptest" "os" "testing" + "path/filepath" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/crowdsecurity/go-cs-lib/cstest" + + "github.com/crowdsecurity/crowdsec/pkg/csconfig" ) -func TestInitHubUpdate(t *testing.T) { - hub := envSetup(t) +// testHubCfg creates an empty hub structure in a temporary directory +// and returns its configuration object. +// +// This allow the reuse of the temporary directory / hub content for multiple instances +// of the Hub object. +func testHubCfg(t *testing.T) *csconfig.LocalHubCfg { + tempDir := t.TempDir() + + local := csconfig.LocalHubCfg{ + HubDir: filepath.Join(tempDir, "crowdsec", "hub"), + HubIndexFile: filepath.Join(tempDir, "crowdsec", "hub", ".index.json"), + InstallDir: filepath.Join(tempDir, "crowdsec"), + InstallDataDir: filepath.Join(tempDir, "installed-data"), + } + + err := os.MkdirAll(local.HubDir, 0o755) + require.NoError(t, err) - _, err := NewHub(hub.local, nil) + err = os.MkdirAll(local.InstallDir, 0o755) require.NoError(t, err) - ctx := context.Background() + err = os.MkdirAll(local.InstallDataDir, 0o755) + require.NoError(t, err) + + return &local +} - indexProvider := &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", +func testHub(t *testing.T, localCfg *csconfig.LocalHubCfg, indexJson string) (*Hub, error) { + if localCfg == nil { + localCfg = testHubCfg(t) } - err = hub.Update(ctx, indexProvider, false) + err := os.WriteFile(localCfg.HubIndexFile, []byte(indexJson), 0o644) require.NoError(t, err) + hub, err := NewHub(localCfg, nil) + require.NoError(t, err) err = hub.Load() + return hub, err +} + +func TestIndexEmpty(t *testing.T) { + // an empty hub is valid, and should not have warnings + hub, err := testHub(t, nil, "{}") require.NoError(t, err) + assert.Empty(t, hub.Warnings) } -func TestUpdateIndex(t *testing.T) { - // bad url template - fmt.Println("Test 'bad URL'") +func TestIndexJSON(t *testing.T) { + // but it can't be an empty string + hub, err := testHub(t, nil, "") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) - tmpIndex, err := os.CreateTemp("", "index.json") + // it must be valid json + hub, err = testHub(t, nil, "def not json") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: invalid character 'd' looking for beginning of value. Run 'sudo cscli hub update' to download the index again") + assert.Empty(t, hub.Warnings) + + hub, err = testHub(t, nil, "{") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: unexpected end of JSON input") + assert.Empty(t, hub.Warnings) + + // and by json we mean an object + hub, err = testHub(t, nil, "[]") + cstest.RequireErrorContains(t, err, "invalid hub index: failed to parse index: json: cannot unmarshal array into Go value of type cwhub.HubItems") + assert.Empty(t, hub.Warnings) +} + +func TestIndexUnknownItemType(t *testing.T) { + // Allow unknown fields in the top level object, likely new item types + hub, err := testHub(t, nil, `{"goodies": {}}`) require.NoError(t, err) + assert.Empty(t, hub.Warnings) +} + +func TestHubUpdate(t *testing.T) { + // update an empty hub with a index containing a parser. - // close the file to avoid preventing the rename on windows - err = tmpIndex.Close() + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - t.Cleanup(func() { - os.Remove(tmpIndex.Name()) - }) + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` - hub := envSetup(t) + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + _, err := w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() - hub.local.HubIndexFile = tmpIndex.Name() + ctx := context.Background() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + } + + err = hub.Update(ctx, downloader, true) + require.NoError(t, err) + + err = hub.Load() + require.NoError(t, err) + + item := hub.GetItem("parsers", "author/pars1") + assert.NotEmpty(t, item) + assert.Equal(t, "author/pars1", item.Name) +} + +func TestHubUpdateInvalidTemplate(t *testing.T) { + hub, err := testHub(t, nil, "{}") + require.NoError(t, err) ctx := context.Background() - indexProvider := &Downloader{ + downloader := &Downloader{ + Branch: "main", URLTemplate: "x", - Branch: "", } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to build hub index request: invalid URL template 'x'") + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorMessage(t, err, "failed to build hub index request: invalid URL template 'x'") +} + - // bad domain - fmt.Println("Test 'bad domain'") - indexProvider = &Downloader{ - URLTemplate: "https://baddomain/crowdsecurity/%s/%s", - Branch: "master", - } - err = hub.Update(ctx, indexProvider, false) +func TestHubUpdateCannotWrite(t *testing.T) { + hub, err := testHub(t, nil, "{}") require.NoError(t, err) - // XXX: this is not failing - // cstest.RequireErrorContains(t, err, "failed http request for hub index: Get") - // bad target path - fmt.Println("Test 'bad target path'") + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + _, err := w.Write([]byte(index1)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() - indexProvider = &Downloader{ - URLTemplate: mockURLTemplate, - Branch: "master", + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", } - hub.local.HubIndexFile = "/does/not/exist/index.json" + hub.local.HubIndexFile = "/proc/foo/bar/baz/.index.json" + + err = hub.Update(ctx, downloader, true) + cstest.RequireErrorContains(t, err, "failed to create temporary download file for /proc/foo/bar/baz/.index.json") +} + +func TestHubUpdateAfterLoad(t *testing.T) { + // Update() can't be called after Load() if the hub is not completely empty. + + index1 := ` +{ + "parsers": { + "author/pars1": { + "path": "parsers/s01-parse/pars1.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + hub, err := testHub(t, nil, index1) + require.NoError(t, err) + + index2 := ` +{ + "parsers": { + "author/pars2": { + "path": "parsers/s01-parse/pars2.yaml", + "stage": "s01-parse", + "version": "0.0", + "versions": { + "0.0": { + "digest": "44136fa355b3678a1146ad16f7e8649e94fb4fc21fe77e8310c060f61caaff8a" + } + }, + "content": "{}" + } + } +}` + + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != "/main/.index.json" { + w.WriteHeader(http.StatusNotFound) + } + _, err := w.Write([]byte(index2)) + assert.NoError(t, err) + })) + defer mockServer.Close() + + ctx := context.Background() + + downloader := &Downloader{ + Branch: "main", + URLTemplate: mockServer.URL + "/%s/%s", + } - err = hub.Update(ctx, indexProvider, false) - cstest.RequireErrorContains(t, err, "failed to create temporary download file for /does/not/exist/index.json:") + err = hub.Update(ctx, downloader, true) + require.ErrorIs(t, err, ErrUpdateAfterSync) }