sonatype nexus - quirks modes

docs/reference/ocm_credential-handling.md

@@ -167,7 +167,7 @@ The following credential consumer types are used/supported:
       - <code>password</code>: the basic auth password
 
 
-  - <code>NpmRegistry</code>: NPM repository
+  - <code>NpmRegistry</code>: NPM registry
 
     It matches the <code>NpmRegistry</code> consumer type and additionally acts like
     the <code>hostpath</code> type.

docs/reference/ocm_get_credentials.md

@@ -93,7 +93,7 @@ Matchers exist for the following usage contexts or consumer types:
       - <code>password</code>: the basic auth password
 
 
-  - <code>NpmRegistry</code>: NPM repository
+  - <code>NpmRegistry</code>: NPM registry
 
     It matches the <code>NpmRegistry</code> consumer type and additionally acts like
     the <code>hostpath</code> type.

pkg/contexts/credentials/builtin/npm/identity/identity.go

@@ -1,11 +1,8 @@
 package identity
 
 import (
-	"path"
+	"net/url"
 
-	. "net/url"
-
-	"github.com/open-component-model/ocm/pkg/common"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/cpi"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/identity/hostpath"
 	"github.com/open-component-model/ocm/pkg/listformat"
@@ -37,31 +34,35 @@ func init() {
 		ATTR_TOKEN, "the token attribute. May exist after login at any npm registry. Check your .npmrc file!",
 	})
 
-	cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM repository
+	cpi.RegisterStandardIdentity(CONSUMER_TYPE, hostpath.IdentityMatcher(CONSUMER_TYPE), `NPM registry
 
 It matches the <code>`+CONSUMER_TYPE+`</code> consumer type and additionally acts like 
 the <code>`+hostpath.IDENTITY_TYPE+`</code> type.`,
 		attrs)
 }
 
-func GetConsumerId(rawURL string, pkgName string) cpi.ConsumerIdentity {
-	url, err := Parse(rawURL)
+var identityMatcher = hostpath.IdentityMatcher(CONSUMER_TYPE)
+
+func IdentityMatcher(pattern, cur, id cpi.ConsumerIdentity) bool {
+	return identityMatcher(pattern, cur, id)
+}
+
+func GetConsumerId(rawURL, groupId string) (cpi.ConsumerIdentity, error) {
+	_url, err := url.JoinPath(rawURL, groupId)
 	if err != nil {
-		return nil
+		return nil, err
 	}
-
-	url.Path = path.Join(url.Path, pkgName)
-	return hostpath.GetConsumerIdentity(CONSUMER_TYPE, url.String())
+	return hostpath.GetConsumerIdentity(CONSUMER_TYPE, _url), nil
 }
 
-func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) common.Properties {
-	id := GetConsumerId(repoUrl, pkgName)
-	if id == nil {
-		return nil
+func GetCredentials(ctx cpi.ContextProvider, repoUrl string, pkgName string) (cpi.Credentials, error) {
+	id, err := GetConsumerId(repoUrl, pkgName)
+	if err != nil {
+		return nil, err
 	}
-	credentials, err := cpi.CredentialsForConsumer(ctx.CredentialsContext(), id)
-	if credentials == nil || err != nil {
-		return nil
+	if id == nil {
+		logging.DynamicLogger(REALM).Debug("No consumer identity found.", "url", repoUrl, "groupId", pkgName)
+		return nil, nil
 	}
-	return credentials.Properties()
+	return cpi.CredentialsForConsumer(ctx.CredentialsContext(), id)
 }

pkg/contexts/credentials/repositories/npm/config_test.go

@@ -3,31 +3,27 @@ package npm_test
 import (
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	. "github.com/open-component-model/ocm/pkg/testutils"
 
 	"github.com/open-component-model/ocm/pkg/common"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/builtin/npm/identity"
 	"github.com/open-component-model/ocm/pkg/contexts/credentials/repositories/npm"
+	. "github.com/open-component-model/ocm/pkg/testutils"
 )
 
 var _ = Describe("Config deserialization Test Environment", func() {
 	It("read .npmrc", func() {
 		ctx := credentials.New()
-
 		repo := Must(npm.NewRepository(ctx, "testdata/.npmrc"))
 		Expect(Must(repo.LookupCredentials("registry.npmjs.org")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "npm_TOKEN"}))
 		Expect(Must(repo.LookupCredentials("npm.registry.acme.com/api/npm")).Properties()).To(Equal(common.Properties{identity.ATTR_TOKEN: "bearer_TOKEN"}))
 	})
 
 	It("propagates credentials", func() {
 		ctx := credentials.New()
-
 		spec := npm.NewRepositorySpec("testdata/.npmrc")
-
 		_ = Must(ctx.RepositoryForSpec(spec))
-		id := identity.GetConsumerId("registry.npmjs.org", "pkg")
-
+		id := Must(identity.GetConsumerId("registry.npmjs.org", "pkg"))
 		creds := Must(credentials.CredentialsForConsumer(ctx, id))
 		Expect(creds).NotTo(BeNil())
 		Expect(creds.GetProperty(identity.ATTR_TOKEN)).To(Equal("npm_TOKEN"))
@@ -39,5 +35,4 @@ var _ = Describe("Config deserialization Test Environment", func() {
 		Expect(t).NotTo(BeNil())
 		Expect(t.Description()).NotTo(Equal(""))
 	})
-
 })

pkg/contexts/credentials/repositories/npm/provider.go

@@ -35,8 +35,12 @@ func (p *ConsumerProvider) get(requested cpi.ConsumerIdentity, currentFound cpi.
 	var creds cpi.CredentialsSource
 
 	for key, value := range all {
-		id := npm.GetConsumerId("https://"+key, "")
-
+		id, err := npm.GetConsumerId("https://"+key, "")
+		if err != nil {
+			log := logging.Context().Logger(npm.REALM)
+			log.LogError(err, "Failed to get consumer id", "key", key, "value", value)
+			return nil, nil
+		}
 		if m(requested, currentFound, id) {
 			creds = newCredentials(value)
 			currentFound = id

pkg/contexts/credentials/repositories/npm/repository.go

@@ -65,7 +65,7 @@ func (r *Repository) Read(force bool) error {
 	}
 
 	if r.path == "" {
-		return fmt.Errorf("npmrc path not provided")
+		return errors.New("npmrc path not provided")
 	}
 	cfg, path, err := readNpmConfigFile(r.path)
 	if err != nil {

pkg/contexts/ocm/accessmethods/npm/method.go

@@ -86,7 +86,7 @@ func (a *AccessSpec) AccessMethod(c accspeccpi.ComponentVersionAccess) (accspecc
 }
 
 func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.ComponentVersionAccess) string {
-	meta, err := a.getPackageMeta(access.GetContext())
+	meta, err := a.GetPackageVersion(access.GetContext())
 	if err != nil {
 		return ""
 	}
@@ -96,43 +96,70 @@ func (a *AccessSpec) GetInexpensiveContentVersionIdentity(access accspeccpi.Comp
 	return ""
 }
 
-// PackageUrl returns the URL of the NPM package (Registry/Package/Version).
+// PackageUrl returns the URL of the NPM package (Registry/Package).
 func (a *AccessSpec) PackageUrl() string {
-	return a.Registry + path.Join("/", a.Package, a.Version)
+	return strings.TrimSuffix(a.Registry, "/") + path.Join("/", a.Package)
 }
 
-func (a *AccessSpec) getPackageMeta(ctx accspeccpi.Context) (*meta, error) {
+// PackageVersionUrl returns the URL of the NPM package-version (Registry/Package/Version).
+func (a *AccessSpec) PackageVersionUrl() string {
+	return strings.TrimSuffix(a.Registry, "/") + path.Join("/", a.Package, a.Version)
+}
+
+func (a *AccessSpec) GetPackageVersion(ctx accspeccpi.Context) (*npm.Version, error) {
 	r, err := reader(a, vfsattr.Get(ctx), ctx)
 	if err != nil {
 		return nil, err
 	}
-	buf := &bytes.Buffer{}
-	_, err = io.Copy(buf, io.LimitReader(r, 200000))
+	defer r.Close()
+	buf, err := io.ReadAll(r)
 	if err != nil {
-		return nil, errors.Wrapf(err, "cannot get version metadata for %s", a.PackageUrl())
+		return nil, errors.Wrapf(err, "cannot get version metadata for %s", a.PackageVersionUrl())
 	}
-
-	var metadata meta
-
-	err = json.Unmarshal(buf.Bytes(), &metadata)
-	if err != nil {
-		return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", a.PackageUrl())
+	var version npm.Version
+	err = json.Unmarshal(buf, &version)
+	if err != nil || version.Dist.Tarball == "" {
+		// ugly fallback as workaround for https://github.com/sonatype/nexus-public/issues/224
+		var project npm.Project
+		err = json.Unmarshal(buf, &project) // parse the complete project
+		if err != nil {
+			return nil, errors.Wrapf(err, "cannot unmarshal version metadata for %s", a.PackageVersionUrl())
+		}
+		v, ok := project.Version[a.Version] // and pick only the specified version
+		if !ok {
+			return nil, errors.Newf("version '%s' doesn't exist", a.Version)
+		}
+		version = v
 	}
-	return &metadata, nil
+	return &version, nil
 }
 
 ////////////////////////////////////////////////////////////////////////////////
 
 func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.AccessMethodImpl, error) {
 	factory := func() (blobaccess.BlobAccess, error) {
-		meta, err := a.getPackageMeta(c.GetContext())
+		meta, err := a.GetPackageVersion(c.GetContext())
 		if err != nil {
 			return nil, err
 		}
 
 		f := func() (io.ReadCloser, error) {
 			return reader(a, vfsattr.Get(c.GetContext()), c.GetContext(), meta.Dist.Tarball)
 		}
+		if meta.Dist.Integrity != "" {
+			tf := f
+			f = func() (io.ReadCloser, error) {
+				r, err := tf()
+				if err != nil {
+					return nil, err
+				}
+				digest, err := iotools.DecodeBase64ToHex(meta.Dist.Integrity)
+				if err != nil {
+					return nil, err
+				}
+				return iotools.VerifyingReaderWithHash(r, crypto.SHA512, digest), nil
+			}
+		}
 		if meta.Dist.Shasum != "" {
 			tf := f
 			f = func() (io.ReadCloser, error) {
@@ -149,15 +176,8 @@ func newMethod(c accspeccpi.ComponentVersionAccess, a *AccessSpec) (accspeccpi.A
 	return accspeccpi.NewDefaultMethodImpl(c, a, "", mime.MIME_TGZ, factory), nil
 }
 
-type meta struct {
-	Dist struct {
-		Shasum  string `json:"shasum"`
-		Tarball string `json:"tarball"`
-	} `json:"dist"`
-}
-
 func reader(a *AccessSpec, fs vfs.FileSystem, ctx cpi.ContextProvider, tar ...string) (io.ReadCloser, error) {
-	url := a.PackageUrl()
+	url := a.PackageVersionUrl()
 	if len(tar) > 0 {
 		url = tar[0]
 	}
@@ -170,12 +190,38 @@ func reader(a *AccessSpec, fs vfs.FileSystem, ctx cpi.ContextProvider, tar ...st
 	if err != nil {
 		return nil, err
 	}
-	npm.Authorize(req, ctx, a.Registry, a.Package)
+	err = npm.BasicAuth(req, ctx, a.Registry, a.Package)
+	if err != nil {
+		return nil, err
+	}
 	c := &http.Client{}
 	resp, err := c.Do(req)
 	if err != nil {
 		return nil, err
 	}
+	defer resp.Body.Close()
+	if resp.StatusCode == http.StatusNotFound {
+		// maybe it's stupid Nexus - https://github.com/sonatype/nexus-public/issues/224?
+		url = a.PackageUrl()
+		req, err = http.NewRequestWithContext(context.Background(), http.MethodGet, url, nil)
+		if err != nil {
+			return nil, err
+		}
+		err = npm.BasicAuth(req, ctx, a.Registry, a.Package)
+		if err != nil {
+			return nil, err
+		}
+
+		// close body before overwriting to close any pending connections
+		resp.Body.Close()
+		resp, err = c.Do(req)
+		if err != nil {
+			return nil, err
+		}
+
+		defer resp.Body.Close()
+	}
+
 	if resp.StatusCode != http.StatusOK {
 		defer resp.Body.Close()
 		buf := &bytes.Buffer{}
@@ -185,5 +231,9 @@ func reader(a *AccessSpec, fs vfs.FileSystem, ctx cpi.ContextProvider, tar ...st
 		}
 		return nil, errors.Newf("version meta data request %s provides %s: %s", url, resp.Status, buf.String())
 	}
-	return resp.Body, nil
+	content, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	return io.NopCloser(bytes.NewBuffer(content)), nil
 }

pkg/contexts/ocm/accessmethods/npm/method_test.go

@@ -5,15 +5,15 @@ import (
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
-	. "github.com/open-component-model/ocm/pkg/env"
-	. "github.com/open-component-model/ocm/pkg/env/builder"
-	. "github.com/open-component-model/ocm/pkg/testutils"
 
 	"github.com/open-component-model/ocm/pkg/contexts/ocm"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/npm"
 	"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
+	. "github.com/open-component-model/ocm/pkg/env"
+	. "github.com/open-component-model/ocm/pkg/env/builder"
 	"github.com/open-component-model/ocm/pkg/iotools"
 	"github.com/open-component-model/ocm/pkg/mime"
+	. "github.com/open-component-model/ocm/pkg/testutils"
 )
 
 const NPMPATH = "/testdata/registry"
@@ -62,4 +62,20 @@ var _ = Describe("Method", func() {
 		_, err := m.Reader()
 		Expect(err).To(MatchError(ContainSubstring("SHA-1 digest mismatch: expected 44a77645201d1a8fc5213ace787c220eabbd0967, found 34a77645201d1a8fc5213ace787c220eabbd0967")))
 	})
+
+	It("PackageUrl()", func() {
+		packageUrl := "https://registry.npmjs.org/yargs"
+		acc := npm.New("https://registry.npmjs.org", "yargs", "17.7.1")
+		Expect(acc.PackageUrl()).To(Equal(packageUrl))
+		acc = npm.New("https://registry.npmjs.org/", "yargs", "17.7.1")
+		Expect(acc.PackageUrl()).To(Equal(packageUrl))
+	})
+
+	It("PackageVersionUrl()", func() {
+		packageVersionUrl := "https://registry.npmjs.org/yargs/17.7.1"
+		acc := npm.New("https://registry.npmjs.org", "yargs", "17.7.1")
+		Expect(acc.PackageVersionUrl()).To(Equal(packageVersionUrl))
+		acc = npm.New("https://registry.npmjs.org/", "yargs", "17.7.1")
+		Expect(acc.PackageVersionUrl()).To(Equal(packageVersionUrl))
+	})
 })