Skip to content

Commit

Permalink
fix maven issues (#790)
Browse files Browse the repository at this point in the history
* add fix for empty url

* fix and test maven upload handler

* fix and test maven upload handler

* fix makefile typo
  • Loading branch information
fabianburth authored Jun 4, 2024
1 parent 49a79ea commit 4f6c9eb
Show file tree
Hide file tree
Showing 20 changed files with 309 additions and 73 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ test-all: install-requirements

.PHONY: generate
generate:
@$(REPO_ROOT)/hack/generate.sh $(REPO_ROOT)/pkg... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/helminstaller/... $(REPO_ROOT)/examples/...
@$(REPO_ROOT)/hack/generate.sh $(REPO_ROOT)/pkg/... $(REPO_ROOT)/cmds/ocm/... $(REPO_ROOT)/cmds/helminstaller/... $(REPO_ROOT)/examples/...

.PHONY: generate-deepcopy
generate-deepcopy: controller-gen
Expand Down
4 changes: 4 additions & 0 deletions cmds/ocm/commands/ocmcmds/common/addhdlrs/rscs/elements.go
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,10 @@ type ResourceSpec struct {

var _ addhdlrs.ElementSpec = (*ResourceSpec)(nil)

func (r *ResourceSpec) GetType() string {
return r.Type
}

func (r *ResourceSpec) GetRawIdentity() metav1.Identity {
return r.ElementMeta.GetRawIdentity()
}
Expand Down
4 changes: 4 additions & 0 deletions cmds/ocm/commands/ocmcmds/common/inputs/types/maven/spec.go
Original file line number Diff line number Diff line change
Expand Up @@ -95,5 +95,9 @@ func (s *Spec) GetBlob(ctx inputs.Context, info inputs.InputResourceInfo) (bloba
mavenblob.WithCachingFileSystem(vfsattr.Get(ctx)),
)

if s.IsPackage() {
return access, s.GAV(), err
}

return access, "", err
}
9 changes: 8 additions & 1 deletion cmds/ocm/commands/ocmcmds/components/add/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package add

import (
"fmt"
"github.com/open-component-model/ocm/cmds/ocm/commands/ocmcmds/common/options/uploaderoption"

"github.com/mandelsoft/goutils/errors"
"github.com/mandelsoft/goutils/general"
Expand Down Expand Up @@ -64,7 +65,8 @@ func NewCommand(ctx clictx.Context, names ...string) *cobra.Command {
templateroption.New(""),
dryrunoption.New("evaluate and print component specifications", true),
lookupoption.New(),
rscbyvalueoption.New()),
rscbyvalueoption.New(),
uploaderoption.New(ctx.OCMContext())),
}, utils.Names(Names, names...)...)
}

Expand Down Expand Up @@ -183,6 +185,11 @@ func (o *Command) Complete(args []string) error {
return accessio.ErrInvalidFileFormat(format.String())
}

err = uploaderoption.From(o).Register(o)
if err != nil {
return err
}

return nil
}

Expand Down
123 changes: 123 additions & 0 deletions cmds/ocm/coretests/maven/cmd_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,123 @@
package add_test

import (
. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
. "github.com/open-component-model/ocm/cmds/ocm/testhelper"
"github.com/open-component-model/ocm/pkg/contexts/ocm"
"github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/localblob"
mavenacc "github.com/open-component-model/ocm/pkg/contexts/ocm/accessmethods/maven"
"github.com/open-component-model/ocm/pkg/maven"
. "github.com/open-component-model/ocm/pkg/testutils"
"strings"
"time"

"github.com/open-component-model/ocm/pkg/contexts/ocm/repositories/ctf"
"github.com/open-component-model/ocm/pkg/maven/maventest"
)

const (
MAVEN_CENTRAL_ADDRESS = "repo.maven.apache.org:443"
MAVEN_CENTRAL = "https://repo.maven.apache.org/maven2/"
MAVEN_GROUP_ID = "maven"
MAVEN_ARTIFACT_ID = "maven"
MAVEN_VERSION = "1.1"
)

const ARCH = "/tmp/ctf"
const DEST_ARCH = "/tmp/ctf-dest"
const VERSION = "1.0.0"
const COMPONENT = "ocm.software/demo/test"
const OUT = "/tmp/res"

var _ = Describe("Test Environment", func() {
var env *TestEnv

BeforeEach(func() {
env = NewTestEnv(TestData(), maventest.TestData("/maven/testdata"))
})

AfterEach(func() {
env.Cleanup()
})

It("upload maven package from localblob during transfer", func() {
coords := maven.NewCoordinates(maventest.GROUP_ID, maventest.ARTIFACT_ID, maventest.VERSION)
Expect(env.Execute("add", "cv", "-fc", "--file", ARCH, "testdata/components.yaml")).To(Succeed())
Expect(env.DirExists(ARCH)).To(BeTrue())
repo := Must(ctf.Open(env, ctf.ACC_READONLY, ARCH, 0, env))
defer Close(repo)
cv := Must(repo.LookupComponentVersion(COMPONENT, VERSION))
defer Close(cv)
Expect(len(cv.GetDescriptor().Resources)).To(Equal(1))
acc := Must(env.OCMContext().AccessSpecForSpec(cv.GetDescriptor().Resources[0].Access))
Expect(acc.IsLocal(env.OCMContext())).To(BeTrue())
Expect(acc.(*localblob.AccessSpec).ReferenceName).To(Equal(strings.Join([]string{maventest.GROUP_ID, maventest.ARTIFACT_ID, maventest.VERSION}, ":")))

Expect(env.Execute("transfer", "ctf", ARCH, DEST_ARCH, "--uploader", "ocm/mavenArtifact=file://localhost/mavenrepo")).To(Succeed())
Expect(env.DirExists(DEST_ARCH)).To(BeTrue())
Expect(env.DirExists("/mavenrepo/" + coords.GavPath())).To(BeTrue())
mavenrepo := maven.NewFileRepository("/mavenrepo", env.FileSystem())
Expect(mavenrepo.GavFiles(coords, nil)).To(YAMLEqual(`
sdk-modules-bom-5.7.0-random-content.json: 5
sdk-modules-bom-5.7.0-random-content.txt: 5
sdk-modules-bom-5.7.0-sources.jar: 5
sdk-modules-bom-5.7.0.jar: 5
sdk-modules-bom-5.7.0.pom: 5`))
})

It("upload maven package from localblob during component composition", func() {
coords := maven.NewCoordinates(maventest.GROUP_ID, maventest.ARTIFACT_ID, maventest.VERSION)
Expect(env.Execute("add", "cv", "-fc", "--file", ARCH, "testdata/components.yaml", "--uploader", "ocm/mavenArtifact=file://localhost/mavenrepo")).To(Succeed())
Expect(env.DirExists(ARCH)).To(BeTrue())
repo := Must(ctf.Open(env, ctf.ACC_READONLY, ARCH, 0, env))
defer Close(repo)
cv := Must(repo.LookupComponentVersion(COMPONENT, VERSION))
defer Close(cv)
Expect(len(cv.GetDescriptor().Resources)).To(Equal(1))
acc := Must(env.OCMContext().AccessSpecForSpec(cv.GetDescriptor().Resources[0].Access))
Expect(acc.IsLocal(env.OCMContext())).To(BeFalse())
Expect(acc.GetKind()).To(Equal(mavenacc.Type))
Expect(acc.(*mavenacc.AccessSpec).GAV()).To(Equal(strings.Join([]string{maventest.GROUP_ID, maventest.ARTIFACT_ID, maventest.VERSION}, ":")))

Expect(env.DirExists("/mavenrepo/" + coords.GavPath())).To(BeTrue())
mavenrepo := maven.NewFileRepository("/mavenrepo", env.FileSystem())
Expect(mavenrepo.GavFiles(coords, nil)).To(YAMLEqual(`
sdk-modules-bom-5.7.0-random-content.json: 5
sdk-modules-bom-5.7.0-random-content.txt: 5
sdk-modules-bom-5.7.0-sources.jar: 5
sdk-modules-bom-5.7.0.jar: 5
sdk-modules-bom-5.7.0.pom: 5`))
})

Context("maven http repository", func() {
if PingTCPServer(MAVEN_CENTRAL_ADDRESS, time.Second) == nil {
var coords *maven.Coordinates
BeforeEach(func() {
coords = maven.NewCoordinates(MAVEN_GROUP_ID, MAVEN_ARTIFACT_ID, MAVEN_VERSION)
})
It("upload maven package from access method", func() {
Expect(env.Execute("add", "cv", "-fc", "--file", ARCH, "testdata/components2.yaml")).To(Succeed())
Expect(env.DirExists(ARCH)).To(BeTrue())
repo := Must(ctf.Open(env, ctf.ACC_READONLY, ARCH, 0, env))
defer Close(repo)
cv := Must(repo.LookupComponentVersion(COMPONENT, VERSION))
defer Close(cv)
Expect(len(cv.GetDescriptor().Resources)).To(Equal(1))
acc := Must(env.OCMContext().AccessSpecForSpec(cv.GetDescriptor().Resources[0].Access))
Expect(acc.IsLocal(env.OCMContext())).To(BeFalse())
Expect(acc.(ocm.HintProvider).GetReferenceHint(cv)).To(Equal(coords.GAV()))

Expect(env.Execute("transfer", "ctf", ARCH, DEST_ARCH, "--copy-resources", "--uploader", "ocm/mavenArtifact=file://localhost/mavenrepo")).To(Succeed())
Expect(env.DirExists(DEST_ARCH)).To(BeTrue())
Expect(env.DirExists("/mavenrepo/" + coords.GavPath())).To(BeTrue())
mavenrepo := maven.NewFileRepository("/mavenrepo", env.FileSystem())
Expect(mavenrepo.GavFiles(coords, nil)).To(YAMLEqual(`
maven-1.1-RC1.javadoc.javadoc.jar: 5
maven-1.1-sources.jar: 5
maven-1.1.jar: 5
maven-1.1.pom: 5`))
})
}
})
})
13 changes: 13 additions & 0 deletions cmds/ocm/coretests/maven/suite_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
package add_test

import (
"testing"

. "github.com/onsi/ginkgo/v2"
. "github.com/onsi/gomega"
)

func TestConfig(t *testing.T) {
RegisterFailHandler(Fail)
RunSpecs(t, "OCM add components")
}
14 changes: 14 additions & 0 deletions cmds/ocm/coretests/maven/testdata/components.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,14 @@
components:
- name: ocm.software/demo/test
version: 1.0.0
provider:
name: ocm.software
resources:
- name: mavenartifact
type: mavenArtifact
input:
type: maven
path: /maven/testdata/.m2/repository
groupId: "com.sap.cloud.sdk"
artifactId: "sdk-modules-bom"
version: "5.7.0"
15 changes: 15 additions & 0 deletions cmds/ocm/coretests/maven/testdata/components2.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
components:
- name: ocm.software/demo/test
version: 1.0.0
provider:
name: ocm.software
resources:
- name: mavenartifact
type: mavenArtifact
version: "1.1"
access:
type: maven
repoUrl: "https://repo.maven.apache.org/maven2/"
groupId: "maven"
artifactId: "maven"
version: "1.1"
4 changes: 2 additions & 2 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -39,8 +39,8 @@ require (
github.com/klauspost/compress v1.17.8
github.com/klauspost/pgzip v1.2.6
github.com/mandelsoft/filepath v0.0.0-20240223090642-3e2777258aa3
github.com/mandelsoft/goutils v0.0.0-20240527090454-525d51156f92
github.com/mandelsoft/logging v0.0.0-20240326140403-99e2fb8bdce6
github.com/mandelsoft/goutils v0.0.0-20240604075441-f06e2890eea3
github.com/mandelsoft/logging v0.0.0-20240201091719-67180059d6bf
github.com/mandelsoft/spiff v1.7.0-beta-5
github.com/mandelsoft/vfs v0.4.3
github.com/marstr/guid v1.1.0
Expand Down
8 changes: 4 additions & 4 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -682,10 +682,10 @@ github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0
github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc=
github.com/mandelsoft/filepath v0.0.0-20240223090642-3e2777258aa3 h1:oo9nIgnyiBgYPbcZslRT4y29siuL5EoNJ/t1tr0xEVQ=
github.com/mandelsoft/filepath v0.0.0-20240223090642-3e2777258aa3/go.mod h1:LxhqC7khDoRENwooP6f/vWvia9ivj6TqLYrR39zqkN0=
github.com/mandelsoft/goutils v0.0.0-20240527090454-525d51156f92 h1:JKHpPtPpkCA7AVRv5trXmqAcncjbfkjiv4x7wGSJnEc=
github.com/mandelsoft/goutils v0.0.0-20240527090454-525d51156f92/go.mod h1:EbNqk9JceSMq7MJuALB/vlOpoD4MAGE0TenM9TR+C0o=
github.com/mandelsoft/logging v0.0.0-20240326140403-99e2fb8bdce6 h1:xLaVUuFrXqcAZJ2rGZdRHX1Qe4AvxZINJ5XAXKmL2uc=
github.com/mandelsoft/logging v0.0.0-20240326140403-99e2fb8bdce6/go.mod h1:uO460C1lIB3IOOgrbXhAlz3AKsOv4T2K6ALBn3PwuSg=
github.com/mandelsoft/goutils v0.0.0-20240604075441-f06e2890eea3 h1:jSXyDjsOuy0DUCzDUCrZS/QMJZ2we1HQfLCpb0D7cnY=
github.com/mandelsoft/goutils v0.0.0-20240604075441-f06e2890eea3/go.mod h1:EbNqk9JceSMq7MJuALB/vlOpoD4MAGE0TenM9TR+C0o=
github.com/mandelsoft/logging v0.0.0-20240201091719-67180059d6bf h1:WEmgzeArDbp6Aw34jmziMIE5ygo2zpl/atXRq3D7lSw=
github.com/mandelsoft/logging v0.0.0-20240201091719-67180059d6bf/go.mod h1:uO460C1lIB3IOOgrbXhAlz3AKsOv4T2K6ALBn3PwuSg=
github.com/mandelsoft/spiff v1.7.0-beta-5 h1:3kC10nTviDQhL8diSxp7i4IC2iSiDg6KPbH1CAq7Lfw=
github.com/mandelsoft/spiff v1.7.0-beta-5/go.mod h1:TwEeOPuRZxlzQBCLEyVTlHmBSruSGGNdiQ2fovVJ8ao=
github.com/mandelsoft/vfs v0.4.3 h1:2UMrxQkMXkcHyuqSFhgFDupQ1fmqpKLZuu04DOHx1PA=
Expand Down
10 changes: 4 additions & 6 deletions pkg/blobaccess/maven/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -22,10 +22,8 @@ type Options struct {
CachingPath string
// Credentials allows to pass credentials and certificates for the http communication
Credentials credentials.Credentials
// Classifier defines the classifier of the maven file coordinates
Classifier *string
// Extension defines the extension of the maven file coordinates
Extension *string

maven.FileCoordinates
}

func (o *Options) Logger(keyValuePairs ...interface{}) logging.Logger {
Expand Down Expand Up @@ -179,7 +177,7 @@ func WithOptionalClassifier(c *string) Option {
if c != nil {
return WithClassifier(*c)
}
return nil
return &optionutils.NoOption[*Options]{}
}

////////////////////////////////////////////////////////////////////////////////
Expand All @@ -198,7 +196,7 @@ func WithOptionalExtension(e *string) Option {
if e != nil {
return WithExtension(*e)
}
return nil
return &optionutils.NoOption[*Options]{}
}

////////////////////////////////////////////////////////////////////////////////
5 changes: 4 additions & 1 deletion pkg/contexts/ocm/accessmethods/maven/method.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,7 +89,10 @@ func (a *AccessSpec) GlobalAccessSpec(_ accspeccpi.Context) accspeccpi.AccessSpe

// GetReferenceHint returns the reference hint for the Maven (mvn) artifact.
func (a *AccessSpec) GetReferenceHint(_ accspeccpi.ComponentVersionAccess) string {
return a.String()
if a.IsPackage() {
return a.GAV()
}
return ""
}

func (_ *AccessSpec) GetType() string {
Expand Down
4 changes: 4 additions & 0 deletions pkg/contexts/ocm/accessmethods/maven/method_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -60,6 +60,10 @@ var _ = Describe("local accessmethods.maven.AccessSpec tests", func() {
Expect(dr.Size()).To(Equal(int64(maventest.ARTIFACT_SIZE)))
Expect(dr.Digest().String()).To(Equal("SHA-256:" + maventest.ARTIFACT_DIGEST))
})
It("test empty repoUrl", func() {
acc := me.New("", "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0")
ExpectError(acc.AccessMethod(cv)).ToNot(BeNil())
})

It("accesses local artifact with empty classifier and with extension", func() {
acc := me.New("file://"+MAVEN_PATH, "com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0", me.WithClassifier(""), me.WithExtension("pom"))
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -60,7 +60,7 @@ func (b *artifactHandler) StoreBlob(blob cpi.BlobAccess, resourceType string, hi
if err != nil {
return nil, err
}
if coords.Classifier != nil || coords.Extension != nil {
if !coords.IsPackage() {
return nil, nil
}
log = log.WithValues("groupId", coords.GroupId, "artifactId", coords.ArtifactId, "version", coords.Version)
Expand Down
11 changes: 11 additions & 0 deletions pkg/contexts/ocm/elements/artifactblob/mavenblob/access_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,7 @@ var _ = Describe("blobaccess for maven", func() {
maven.WithClassifier("random-content"), maven.WithExtension("json"))

a := me.ResourceAccessForMavenCoords(env.OCMContext(), Must(elements.ResourceMeta("mavenblob", resourcetypes.OCM_JSON, elements.WithLocalRelation())), repo, coords, me.WithCachingFileSystem(env.FileSystem()))
Expect(a.ReferenceHint()).To(Equal(""))
b := Must(a.BlobAccess())
defer Close(b)
Expect(string(Must(b.Get()))).To(Equal(`{"some": "test content"}`))
Expand All @@ -57,5 +58,15 @@ var _ = Describe("blobaccess for maven", func() {
defer Close(m)
Expect(string(Must(m.Get()))).To(Equal(`{"some": "test content"}`))
})

It("blobaccess for package", func() {
cv := composition.NewComponentVersion(env.OCMContext(), "acme.org/test", "1.0.0")
defer Close(cv)

coords := maven.NewCoordinates("com.sap.cloud.sdk", "sdk-modules-bom", "5.7.0")

a := me.ResourceAccessForMavenCoords(env.OCMContext(), Must(elements.ResourceMeta("mavenblob", resourcetypes.OCM_JSON, elements.WithLocalRelation())), repo, coords, me.WithCachingFileSystem(env.FileSystem()))
Expect(a.ReferenceHint()).To(Equal(coords.GAV()))
})
})
})
8 changes: 8 additions & 0 deletions pkg/contexts/ocm/elements/artifactblob/mavenblob/options.go
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
"github.com/open-component-model/ocm/pkg/contexts/datacontext"
"github.com/open-component-model/ocm/pkg/contexts/ocm/cpi"
"github.com/open-component-model/ocm/pkg/contexts/ocm/elements/artifactblob/api"
"github.com/open-component-model/ocm/pkg/maven"
)

type Option = optionutils.Option[*Options]
Expand Down Expand Up @@ -40,6 +41,13 @@ func WithHint(h string) Option {
return api.WrapHint[Options](h)
}

func WithHintForCoords(coords *maven.Coordinates) Option {
if coords.IsPackage() {
return WithHint(coords.GAV())
}
return optionutils.NoOption[*Options]{}
}

func WithGlobalAccess(a cpi.AccessSpec) Option {
return api.WrapGlobalAccess[Options](a)
}
Expand Down
3 changes: 3 additions & 0 deletions pkg/contexts/ocm/elements/artifactblob/mavenblob/resource.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,9 @@ const TYPE = resourcetypes.MAVEN_ARTIFACT

func Access[M any, P compdesc.ArtifactMetaPointer[M]](ctx ocm.Context, meta P, repo *maven.Repository, groupId, artifactId, version string, opts ...Option) cpi.ArtifactAccess[M] {
eff := optionutils.EvalOptions(optionutils.WithDefaults(opts, WithCredentialContext(ctx))...)
if eff.Blob.IsPackage() && eff.Hint == "" {
eff.Hint = maven.NewCoordinates(groupId, artifactId, version).GAV()
}

if meta.GetType() == "" {
meta.SetType(TYPE)
Expand Down
2 changes: 1 addition & 1 deletion pkg/maven/access.go
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ func NewFileRepository(path string, fss ...vfs.FileSystem) *Repository {
}

func NewUrlRepository(repoUrl string, fss ...vfs.FileSystem) (*Repository, error) {
u, err := url.Parse(repoUrl)
u, err := url.ParseRequestURI(repoUrl)
if err != nil {
return nil, err
}
Expand Down
Loading

0 comments on commit 4f6c9eb

Please sign in to comment.