diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..deae4f6 --- /dev/null +++ b/.env.example @@ -0,0 +1,9 @@ +S3_MOCK=0 +AUTH_LEVEL=0 # Options: [0, 1] corresponds to [no FGAC, FGAC]. +AUTH_LIMITED_WRITER_ROLE='s3_limited_writer' + +DBUSER=user +DBPASS=password +DBHOST=localhost +DBPORT=5432 +DBNAME= \ No newline at end of file diff --git a/.example.env.json b/.example.env.json new file mode 100644 index 0000000..9f896e2 --- /dev/null +++ b/.example.env.json @@ -0,0 +1,15 @@ +{ + "accounts": [ + { + "AWS_ACCESS_KEY_ID": "", + "AWS_SECRET_ACCESS_KEY": "" + }, + { + "AWS_ACCESS_KEY_ID": "", + "AWS_SECRET_ACCESS_KEY": "" + } + ], + "bucket_allow_list": [ + "*" + ] # "*" for allowing access to all buckets +} \ No newline at end of file diff --git a/.gitignore b/.gitignore index 9461d55..c547b13 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,27 @@ -# Environments -*.env -.venv -env/ -venv/ -ENV/ -env.bak/ -venv.bak/ +# If you prefer the allow list template instead of the deny list, see community template: +# https://github.com/github/gitignore/blob/main/community/Golang/Go.AllowList.gitignore +# +# Binaries for programs and plugins +*.exe +*.exe~ +*.dll +*.so +*.dylib +# Test binary, built with `go test -c` +*.test + +# Output of the go coverage tool, specifically when used with LiteIDE +*.out + +# Dependency directories (remove the comment below to include it) +# vendor/ + +# Go workspace file +go.work +.env main -.vscode/ \ No newline at end of file +.env.json +.data +.env.json +.vscode/settings.json diff --git a/README.md b/README.md index 17a5ad5..9a3d4aa 100644 --- a/README.md +++ b/README.md @@ -29,24 +29,6 @@ The MCAT includes: ### Getting Started ---- - -- Add a .env file to the root level of this directory with the following structure: - -For local: - -``` -STORE_TYPE='LOCAL' -``` - -For S3: - -``` -STORE_TYPE='S3' -AWS_ACCESS_KEY_ID='**************' -AWS_SECRET_ACCESS_KEY='**************' -AWS_DEFAULT_REGION='us-east-1' -S3_BUCKET='******' ``` - Select the stage in `docker-compose.yml` file @@ -82,3 +64,4 @@ _For example: `http://mcat-ras:5600/isamodel?definition_file=models/ras/CHURCH H To view docs goto: http://localhost:5600/swagger/index.html ![](docs/swagger_image.png) +``` diff --git a/config/config.go b/config/config.go index 4b101cb..9759f9e 100644 --- a/config/config.go +++ b/config/config.go @@ -3,14 +3,16 @@ package config import ( "fmt" "os" + "strconv" - "github.com/USACE/filestore" + "github.com/Dewberry/s3api/blobstore" + "github.com/labstack/gommon/log" ) type APIConfig struct { Host string Port int - FileStore *filestore.FileStore + Bh *blobstore.BlobHandler DestinationCRS int } @@ -24,34 +26,29 @@ func Init() *APIConfig { config := new(APIConfig) config.Host = "" // 0.0.0.0 config.Port = 5600 - config.FileStore = FileStoreInit(os.Getenv("STORE_TYPE")) + config.Bh = blobstoreInit() config.DestinationCRS = 4326 return config } // FileStoreInit initializes the filestore object -func FileStoreInit(store string) *filestore.FileStore { - - var fs filestore.FileStore +func blobstoreInit() *blobstore.BlobHandler { + var authLvl int var err error - switch store { - case "LOCAL": - fs, err = filestore.NewFileStore(filestore.BlockFSConfig{}) - if err != nil { - panic(err) - } - case "S3": - config := filestore.S3FSConfig{ - S3Id: os.Getenv("AWS_ACCESS_KEY_ID"), - S3Key: os.Getenv("AWS_SECRET_ACCESS_KEY"), - S3Region: os.Getenv("AWS_DEFAULT_REGION"), - S3Bucket: os.Getenv("S3_BUCKET"), - } - - fs, err = filestore.NewFileStore(config) + authLvlString := os.Getenv("AUTH_LEVEL") + if authLvlString == "" { + authLvl = 0 + log.Warn("Fine Grained Access Control disabled") + } else { + authLvl, err = strconv.Atoi(authLvlString) if err != nil { - panic(err) + log.Fatalf("could not convert AUTH_LEVEL env variable to integer: %v", err) } } - return &fs + bh, err := blobstore.NewBlobHandler(".env.json", authLvl) + if err != nil { + errMsg := fmt.Errorf("failed to initialize a blobhandler %s", err.Error()) + log.Fatal(errMsg) + } + return bh } diff --git a/go.mod b/go.mod index 6f7d2e8..d89c67e 100644 --- a/go.mod +++ b/go.mod @@ -9,16 +9,17 @@ require ( github.com/go-errors/errors v1.4.1 github.com/jackc/pgx v3.6.2+incompatible github.com/jmoiron/sqlx v1.3.4 - github.com/labstack/echo/v4 v4.3.0 + github.com/labstack/echo/v4 v4.11.1 github.com/pzsz/voronoi v0.0.0-20130609164533-4314be88c79f github.com/swaggo/swag v1.7.0 ) require ( + github.com/Dewberry/s3api v0.0.0-20240112165810-9368b22490e0 // indirect github.com/KyleBanks/depth v1.2.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect - github.com/aws/aws-sdk-go v1.38.68 // indirect + github.com/aws/aws-sdk-go v1.44.315 // indirect github.com/cockroachdb/apd v1.1.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -26,23 +27,27 @@ require ( github.com/go-openapi/spec v0.20.0 // indirect github.com/go-openapi/swag v0.19.12 // indirect github.com/gofrs/uuid v4.0.0+incompatible // indirect + github.com/golang-jwt/jwt v3.2.2+incompatible // indirect + github.com/golang-jwt/jwt/v5 v5.0.0 // indirect github.com/google/uuid v1.2.0 // indirect github.com/jackc/fake v0.0.0-20150926172116-812a484cc733 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/labstack/gommon v0.3.0 // indirect + github.com/labstack/gommon v0.4.0 // indirect + github.com/lib/pq v1.10.9 // indirect github.com/mailru/easyjson v0.7.6 // indirect - github.com/mattn/go-colorable v0.1.8 // indirect - github.com/mattn/go-isatty v0.0.13 // indirect + github.com/mattn/go-colorable v0.1.13 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/shopspring/decimal v1.2.0 // indirect + github.com/sirupsen/logrus v1.9.3 // indirect github.com/valyala/bytebufferpool v1.0.0 // indirect - github.com/valyala/fasttemplate v1.2.1 // indirect - golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e // indirect - golang.org/x/net v0.0.0-20210614182718-04defd469f4e // indirect - golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 // indirect - golang.org/x/text v0.3.6 // indirect - golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 // indirect - golang.org/x/tools v0.0.0-20201207182000-5679438983bd // indirect + github.com/valyala/fasttemplate v1.2.2 // indirect + golang.org/x/crypto v0.17.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect + golang.org/x/time v0.3.0 // indirect + golang.org/x/tools v0.6.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect ) diff --git a/go.sum b/go.sum index f81fe6b..7ef5c20 100644 --- a/go.sum +++ b/go.sum @@ -1,4 +1,6 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/Dewberry/s3api v0.0.0-20240112165810-9368b22490e0 h1:e/axE0L24aj05JjPEzDUIwNlp8BE2Cpoayu+XR6cu2c= +github.com/Dewberry/s3api v0.0.0-20240112165810-9368b22490e0/go.mod h1:H4Pl0uljt0qvyvJyes4iTaUNAETHB2qVjrSQtZGVsx8= github.com/KyleBanks/depth v1.2.1 h1:5h8fQADFrWtarTdtDudMmGsC7GPbOAu6RVB3ffsVFHc= github.com/KyleBanks/depth v1.2.1/go.mod h1:jzSb9d0L43HxTQfT+oSA1EEp2q+ne2uh6XgeJcm8brE= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -12,6 +14,8 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/aws/aws-sdk-go v1.31.0/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go v1.38.68 h1:aOG8geU4SohNp659eKBHRBgbqSrZ6jNZlfimIuJAwL8= github.com/aws/aws-sdk-go v1.38.68/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= +github.com/aws/aws-sdk-go v1.44.315 h1:kYTC+Y/bJ9M7QQRvkI/LN5OWvhkIOL/YuFFRhS5QAOo= +github.com/aws/aws-sdk-go v1.44.315/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -43,6 +47,10 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/gofrs/uuid v4.0.0+incompatible h1:1SD/1F5pU8p29ybwgQSwpQk+mwdRrXCYuPhW6m+TnJw= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= +github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= +github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= +github.com/golang-jwt/jwt/v5 v5.0.0 h1:1n1XNM9hk7O9mnQoNBGolZvzebBQ7p93ULHRc28XJUE= +github.com/golang-jwt/jwt/v5 v5.0.0/go.mod h1:pqrtFR0X4osieyHYxtmOUWsAWrfe1Q5UVIyoH402zdk= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -66,10 +74,16 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/labstack/echo/v4 v4.3.0 h1:DCP6cbtT+Zu++K6evHOJzSgA2115cPMuCx0xg55q1EQ= github.com/labstack/echo/v4 v4.3.0/go.mod h1:PvmtTvhVqKDzDQy4d3bWzPjZLzom4iQbAZy2sgZ/qI8= +github.com/labstack/echo/v4 v4.11.1 h1:dEpLU2FLg4UVmvCGPuk/APjlH6GDpbEPti61srUUUs4= +github.com/labstack/echo/v4 v4.11.1/go.mod h1:YuYRTSM3CHs2ybfrL8Px48bO6BAnYIN4l8wSTMP6BDQ= github.com/labstack/gommon v0.3.0 h1:JEeO0bvc78PKdyHxloTKiF8BD5iGrH8T6MSeGvSgob0= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/labstack/gommon v0.4.0 h1:y7cvthEAEbU0yHOf4axH8ZG2NH8knB9iNSoTO8dyIk8= +github.com/labstack/gommon v0.4.0/go.mod h1:uW6kP17uPlLJsD3ijUYn3/M5bAxtlZhMI6m3MFxTMTM= github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/lib/pq v1.10.9 h1:YXG7RB+JIjhP29X+OtkiDnYaXQwpS4JEWq7dtCCRUEw= +github.com/lib/pq v1.10.9/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= @@ -77,11 +91,18 @@ github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.8 h1:c1ghPdyEDarC70ftn0y+A/Ee++9zz8ljHG1b13eJ0s8= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= +github.com/mattn/go-colorable v0.1.11/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= +github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.13 h1:qdl+GuBjcsKKDco5BsxPJlId98mSWNKqYA+Co0SC1yA= github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= +github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= +github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-sqlite3 v1.14.6 h1:dNPt6NO46WmLVt2DLNpwczCmdV5boIZ6g/tlDrlRUbg= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e h1:fD57ERR4JtEqsWbfPhv4DMiApHyliiK5xCTNVSPiaAs= @@ -96,11 +117,14 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/swaggo/swag v1.7.0 h1:5bCA/MTLQoIqDXXyHfOpMeDvL9j68OY/udlK4pQoo4E= @@ -111,15 +135,23 @@ github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyC github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1 h1:TVEnxayobAdVkhQfrfes2IzOB6o+z4roRkPF52WA1u4= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= +github.com/valyala/fasttemplate v1.2.2 h1:lxLXG0uE3Qnshl9QyaK6XJxMXlQZELvChBOCmQD0Loo= +github.com/valyala/fasttemplate v1.2.2/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -130,8 +162,13 @@ golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210614182718-04defd469f4e h1:XpT3nA5TvE525Ne3hInMh6+GETgn27Zfm9dxsThnX2Q= golang.org/x/net v0.0.0-20210614182718-04defd469f4e/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -142,22 +179,46 @@ golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20211103235746-7861aae1554b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 h1:0A+M6Uqn+Eje4kHMK80dtF3JCXC4ykBgQG4Fe06QRhQ= +golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6 h1:Vv0JUPWTyeqUq42B2WJ1FeIDjjvGKoA2Ss+Ts0lAVbs= golang.org/x/time v0.0.0-20210611083556-38a9dc6acbc6/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20201120155355-20be4ac4bd6e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201207182000-5679438983bd h1:aZYo+3GGTb9Pya0Di6t7G0JOwKGb782xQAJlZyVcwII= golang.org/x/tools v0.0.0-20201207182000-5679438983bd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= +golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= @@ -174,5 +235,6 @@ gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/handlers/isamodel.go b/handlers/isamodel.go index 9c4894d..b542f5e 100644 --- a/handlers/isamodel.go +++ b/handlers/isamodel.go @@ -1,13 +1,14 @@ package handlers import ( + "fmt" "net/http" "path/filepath" "strings" "github.com/Dewberry/mcat-ras/tools" + "github.com/Dewberry/s3api/blobstore" - "github.com/USACE/filestore" "github.com/labstack/echo/v4" ) @@ -20,24 +21,30 @@ import ( // @Param definition_file query string true "/models/ras/CHURCH HOUSE GULLY/CHURCH HOUSE GULLY.prj" // @Success 200 {object} bool // @Router /isamodel [get] -func IsAModel(fs *filestore.FileStore) echo.HandlerFunc { +func IsAModel(bh *blobstore.BlobHandler) echo.HandlerFunc { return func(c echo.Context) error { definitionFile := c.QueryParam("definition_file") + bucket := c.QueryParam("bucket") if definitionFile == "" { return c.JSON(http.StatusBadRequest, "Missing query parameter: `definition_file`") } + s3Ctrl, err := bh.GetController(bucket) + if err != nil { + errMsg := fmt.Errorf("error getting S3 controller: %s", err.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) + } - return c.JSON(http.StatusOK, isAModel(fs, definitionFile)) + return c.JSON(http.StatusOK, isAModel(s3Ctrl, bucket, definitionFile)) } } -func isAModel(fs *filestore.FileStore, definitionFile string) bool { +func isAModel(s3Ctrl *blobstore.S3Controller, bucket, definitionFile string) bool { if filepath.Ext(definitionFile) != ".prj" { return false } - firstLine, err := tools.ReadFirstLine(*fs, definitionFile) + firstLine, err := tools.ReadFirstLine(s3Ctrl, bucket, definitionFile) if err != nil { return false } @@ -46,7 +53,7 @@ func isAModel(fs *filestore.FileStore, definitionFile string) bool { return false } - files, err := modFiles(definitionFile, *fs) + files, err := modFiles(s3Ctrl, bucket, definitionFile) if err != nil { return false } diff --git a/handlers/modeltype.go b/handlers/modeltype.go index c189c5a..7932368 100644 --- a/handlers/modeltype.go +++ b/handlers/modeltype.go @@ -1,9 +1,11 @@ package handlers import ( + "fmt" "net/http" - "github.com/USACE/filestore" // warning: replaces standard errors + // warning: replaces standard errors + "github.com/Dewberry/s3api/blobstore" "github.com/labstack/echo/v4" ) @@ -17,15 +19,20 @@ import ( // @Success 200 {string} string "RAS" // @Failure 500 {object} SimpleResponse // @Router /modeltype [get] -func ModelType(fs *filestore.FileStore) echo.HandlerFunc { +func ModelType(bh *blobstore.BlobHandler) echo.HandlerFunc { return func(c echo.Context) error { definitionFile := c.QueryParam("definition_file") + bucket := c.QueryParam("bucket") if definitionFile == "" { return c.JSON(http.StatusBadRequest, "Missing query parameter: `definition_file`") } - - if !isAModel(fs, definitionFile) { + s3Ctrl, err := bh.GetController(bucket) + if err != nil { + errMsg := fmt.Errorf("error getting S3 controller: %s", err.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) + } + if !isAModel(s3Ctrl, bucket, definitionFile) { return c.JSON(http.StatusBadRequest, definitionFile+" is not a valid RAS prj file.") } diff --git a/handlers/modelversion.go b/handlers/modelversion.go index 354f8be..a6b7ce6 100644 --- a/handlers/modelversion.go +++ b/handlers/modelversion.go @@ -2,6 +2,7 @@ package handlers import ( "bufio" + "bytes" "fmt" "net/http" "path/filepath" @@ -9,8 +10,8 @@ import ( "strings" "github.com/Dewberry/mcat-ras/tools" + "github.com/Dewberry/s3api/blobstore" - "github.com/USACE/filestore" // warning: replaces standard errors "github.com/labstack/echo/v4" ) @@ -24,19 +25,25 @@ import ( // @Success 200 {string} string "4.0" // @Failure 500 {object} SimpleResponse // @Router /modelversion [get] -func ModelVersion(fs *filestore.FileStore) echo.HandlerFunc { +func ModelVersion(bh *blobstore.BlobHandler) echo.HandlerFunc { return func(c echo.Context) error { definitionFile := c.QueryParam("definition_file") + bucket := c.QueryParam("bucket") if definitionFile == "" { return c.JSON(http.StatusBadRequest, "Missing query parameter: `definition_file`") } + s3Ctrl, err := bh.GetController(bucket) + if err != nil { + errMsg := fmt.Errorf("error getting S3 controller: %s", err.Error()) + return c.JSON(http.StatusInternalServerError, errMsg.Error()) + } - if !isAModel(fs, definitionFile) { + if !isAModel(s3Ctrl, bucket, definitionFile) { return c.JSON(http.StatusBadRequest, definitionFile+" is not a valid RAS prj file.") } - version, err := getVersions(definitionFile, *fs) + version, err := getVersions(s3Ctrl, bucket, definitionFile) if err != nil { return c.JSON(http.StatusInternalServerError, err.Error()) } @@ -44,39 +51,48 @@ func ModelVersion(fs *filestore.FileStore) echo.HandlerFunc { return c.JSON(http.StatusOK, version) } } - -func modFiles(definitionFile string, fs filestore.FileStore) ([]string, error) { +func modFiles(s3Ctrl *blobstore.S3Controller, bucket, definitionFile string) ([]string, error) { mFiles := make([]string, 0) prefix := filepath.Dir(definitionFile) + "/" - files, err := fs.GetDir(prefix, false) + result, err := s3Ctrl.GetList(bucket, prefix, false) if err != nil { return mFiles, err } - for _, file := range *files { - // get only files that share the same base name or .prj files for projection - // rational behind .prj file is that there can be a shp file in the same level of Hec-RAS - // providing potential projection - if strings.HasPrefix(filepath.Join(file.Path, file.Name), strings.TrimSuffix(definitionFile, "prj")) || filepath.Ext(file.Name) == ".prj" { - mFiles = append(mFiles, filepath.Join(file.Path, file.Name)) + // Process directories + for _, cp := range result.CommonPrefixes { + dirPath := *cp.Prefix + fileName := filepath.Base(dirPath) + if strings.HasPrefix(filepath.Join(dirPath, fileName), strings.TrimSuffix(definitionFile, "prj")) { + mFiles = append(mFiles, filepath.Join(dirPath, fileName)) + } + } + + // Process files + for _, object := range result.Contents { + filePath := *object.Key + fileName := filepath.Base(filePath) + if strings.HasPrefix(filePath, strings.TrimSuffix(definitionFile, "prj")) || filepath.Ext(fileName) == ".prj" { + mFiles = append(mFiles, filePath) } } return mFiles, nil } -func pullVersion(fp string, fs filestore.FileStore) (string, error) { - f, err := fs.GetObject(fp) +func pullVersion(s3Ctrl *blobstore.S3Controller, bucket, fp string) (string, error) { + key := strings.TrimPrefix(fp, "/") + content, err := s3Ctrl.FetchObjectContent(bucket, key) if err != nil { return "", err } - defer f.Close() - sc := bufio.NewScanner(f) + // Create a new reader from the byte slice + reader := bytes.NewReader(content) + sc := bufio.NewScanner(reader) var line string for sc.Scan() { - line = sc.Text() match, err := regexp.MatchString("Program Version=", line) @@ -89,13 +105,17 @@ func pullVersion(fp string, fs filestore.FileStore) (string, error) { } } + if err := sc.Err(); err != nil { + return "", err + } + return "", fmt.Errorf("unable to find program version in file %s", fp) } -func getVersions(definitionFile string, fs filestore.FileStore) (string, error) { +func getVersions(s3Ctrl *blobstore.S3Controller, bucket, definitionFile string) (string, error) { var version string - mFiles, err := modFiles(definitionFile, fs) + mFiles, err := modFiles(s3Ctrl, bucket, definitionFile) if err != nil { return version, err } @@ -107,7 +127,7 @@ func getVersions(definitionFile string, fs filestore.FileStore) (string, error) if tools.RasRE.Plan.MatchString(ext) || tools.RasRE.Geom.MatchString(ext) || tools.RasRE.AllFlow.MatchString(ext) { - ver, err := pullVersion(fp, fs) + ver, err := pullVersion(s3Ctrl, bucket, fp) if err != nil { fmt.Println(err) } else { diff --git a/handlers/ping.go b/handlers/ping.go deleted file mode 100644 index 612dddf..0000000 --- a/handlers/ping.go +++ /dev/null @@ -1,42 +0,0 @@ -package handlers - -import ( - "net/http" - - "github.com/USACE/filestore" - "github.com/labstack/echo/v4" -) - -type SimpleResponse struct { - Status int - Message string - StackTrace string -} - -// Ping godoc -// @Summary Status Check -// @Description Check which services are operational -// @Tags Health Check -// @Accept json -// @Produce json -// @Success 200 {object} SimpleResponse -// @Router /ping [get] -func Ping(fs *filestore.FileStore) echo.HandlerFunc { - return func(c echo.Context) error { - switch (*fs).(type) { - case *filestore.BlockFS: - // fmt.Println("File is local") - return c.JSON(http.StatusOK, map[string]string{"status": "available"}) - - case *filestore.S3FS: - s3FS := (*fs).(*filestore.S3FS) - err := s3FS.Ping() - if err != nil { - return c.JSON(http.StatusInternalServerError, map[string]string{"status": "unavailable"}) - } - // fmt.Println("File is on S3") - return c.JSON(http.StatusOK, map[string]string{"status": "available"}) - } - return c.JSON(http.StatusInternalServerError, map[string]string{"status": "unavailable"}) - } -} diff --git a/main.go b/main.go index 6af8594..ca70c06 100644 --- a/main.go +++ b/main.go @@ -23,7 +23,7 @@ func main() { e.Use(middleware.Gzip()) // HealthCheck - e.GET("/ping", handlers.Ping(appConfig.FileStore)) + e.GET("/ping", appConfig.Bh.Ping) // Swagger // e.GET("/swagger/*", echoSwagger.WrapHandler) @@ -31,9 +31,9 @@ func main() { // ras endpoints // these endpoints create a Ras Model struct from files // and then apply receiver functions to the struct to answer desired question - e.GET("/isamodel", handlers.IsAModel(appConfig.FileStore)) - e.GET("/modeltype", handlers.ModelType(appConfig.FileStore)) - e.GET("/modelversion", handlers.ModelVersion(appConfig.FileStore)) + e.GET("/isamodel", handlers.IsAModel(appConfig.Bh)) + e.GET("/modeltype", handlers.ModelType(appConfig.Bh)) + e.GET("/modelversion", handlers.ModelVersion(appConfig.Bh)) e.GET("/index", handlers.Index(appConfig.FileStore)) e.GET("/isgeospatial", handlers.IsGeospatial(appConfig.FileStore)) e.GET("/geospatialdata", handlers.GeospatialData(appConfig)) diff --git a/tools/project.go b/tools/project.go index 6ff8f4e..499ed28 100644 --- a/tools/project.go +++ b/tools/project.go @@ -2,6 +2,7 @@ package tools import ( "bufio" + "bytes" "crypto/sha256" "fmt" "io" @@ -9,7 +10,7 @@ import ( "regexp" "strings" - "github.com/USACE/filestore" + "github.com/Dewberry/s3api/blobstore" "github.com/go-errors/errors" // warning: replaces standard errors ) @@ -38,17 +39,17 @@ type PrjFileContents struct { Description string //`json:"Description"` } // -func ReadFirstLine(fs filestore.FileStore, fn string) (string, error) { - file, err := fs.GetObject(fn) +func ReadFirstLine(s3ctrl *blobstore.S3Controller, bucket, fn string) (string, error) { + content, err := s3ctrl.FetchObjectContent(bucket, fn) // Use bucket and fn parameters here if err != nil { - fmt.Println("Couldnt open the file", fn) + fmt.Println("Couldn't open the file", fn) return "", errors.Wrap(err, 0) } - defer file.Close() - reader := bufio.NewReader(file) + // Use bytes.NewReader to create a new reader from the byte slice + reader := bufio.NewReader(bytes.NewReader(content)) line, err := reader.ReadString('\n') - if err != nil { + if err != nil && err != io.EOF { // Check for EOF as ReadString('\n') may hit EOF before finding a newline return "", errors.Wrap(err, 0) } return rmNewLineChar(line), nil