Skip to content

Commit

Permalink
Refactor DownloadTextFile (#25)
Browse files Browse the repository at this point in the history
  • Loading branch information
@walkowif
walkowif authored Feb 15, 2024
1 parent 080a923 commit 6a26580
Show file tree
Hide file tree
Showing 2 changed files with 82 additions and 74 deletions.
59 changes: 33 additions & 26 deletions cmd/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ package cmd
import (
"crypto/tls"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"regexp"
Expand All @@ -45,43 +47,48 @@ type GitHubObject struct {
Sha string `json:"sha"`
}

// DownloadTextFile returns HTTP status code for downloaded file, number of bytes
// in downloaded content, and the downloaded content itself as a string.
func DownloadTextFile(url string, parameters map[string]string) (int, int64, string) { // #nosec G402
// DownloadTextFile returns number of bytes in downloaded content,
// the downloaded content itself as a string, and error if any occurred.
func DownloadTextFile(url string, parameters map[string]string) (int64, string, error) { // #nosec G402
tr := &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: true}}
client := &http.Client{Transport: tr}
req, err := http.NewRequest("GET", url, nil)
checkError(err)
if err != nil {
return 0, "", err
}

for k, v := range parameters {
req.Header.Add(k, v)
}

resp, err := client.Do(req)
checkError(err)

if err == nil {
defer resp.Body.Close()
if resp.StatusCode == http.StatusOK {
body, err2 := io.ReadAll(resp.Body)
checkError(err2)
return resp.StatusCode, resp.ContentLength, string(body)
if err2 == nil {
return resp.ContentLength, string(body), nil
}
return 0, "", err2
}
return 0, "", errors.New("Received status code " + fmt.Sprint(resp.StatusCode))
}
return -1, 0, ""
return 0, "", err
}

// GetGitLabProjectAndSha retrieves information about GitLab repository
// (project path, repository name and commit SHA)
// from projectURL GitLab API endpoint.
func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[string]string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) (string, string, string) {
downloadFileFunction func(string, map[string]string) (int64, string, error)) (string, string, string) {
var remoteUsername, remoteRepo, remoteSha string
log.Trace("Downloading data for GitLab project from ", projectURL)
statusCode, _, projectDataResponse := downloadFileFunction(projectURL, token)
if statusCode == 200 {
_, projectDataResponse, err := downloadFileFunction(projectURL, token)
if err == nil {
var projectData GitLabAPIResponse
err := json.Unmarshal([]byte(projectDataResponse), &projectData)
checkError(err)
err2 := json.Unmarshal([]byte(projectDataResponse), &projectData)
checkError(err2)
projectPath := strings.Split(projectData.PathWithNamespace, "/")
projectPathLength := len(projectPath)
remoteUsername = strings.Join(projectPath[:projectPathLength-1], "/")
Expand All @@ -99,8 +106,8 @@ func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin
urlPath = "branches"
}
tagOrBranchURL := projectURL + "/repository/" + urlPath + "/" + remoteRef
statusCode, _, tagOrBranchDataResponse := downloadFileFunction(tagOrBranchURL, token)
if statusCode == 200 {
_, tagOrBranchDataResponse, err := downloadFileFunction(tagOrBranchURL, token)
if err == nil {
var tagOrBranchData GitLabTagOrBranchResponse
err := json.Unmarshal([]byte(tagOrBranchDataResponse), &tagOrBranchData)
checkError(err)
Expand All @@ -114,7 +121,7 @@ func GetGitLabProjectAndSha(projectURL string, remoteRef string, token map[strin

// GetGitHubSha retrieves SHA of the remoteRef from the remoteUsername/remoteRepo GitHub repository.
func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, token map[string]string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) string {
downloadFileFunction func(string, map[string]string) (int64, string, error)) string {
var remoteSha string
log.Trace("Downloading data for GitHub project ", remoteUsername, "/", remoteRepo)
match, errMatch := regexp.MatchString(`v\d+(\.\d+)*`, remoteRef)
Expand All @@ -128,8 +135,8 @@ func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, to
urlPath = "heads"
}
tagOrBranchURL := "https://api.github.com/repos/" + remoteUsername + "/" + remoteRepo + "/git/ref/" + urlPath + "/" + remoteRef
statusCode, _, tagDataResponse := downloadFileFunction(tagOrBranchURL, token)
if statusCode == 200 {
_, tagDataResponse, err := downloadFileFunction(tagOrBranchURL, token)
if err == nil {
var tagOrBranchData GitHubTagOrBranchResponse
err := json.Unmarshal([]byte(tagDataResponse), &tagOrBranchData)
checkError(err)
Expand All @@ -144,7 +151,7 @@ func GetGitHubSha(remoteUsername string, remoteRepo string, remoteRef string, to
// ProcessDescriptionURL gets information about the git repository in which the package is stored
// based on the provided descriptionURL to the package DESCRIPTION file.
func ProcessDescriptionURL(descriptionURL string,
downloadFileFunction func(string, map[string]string) (int, int64, string),
downloadFileFunction func(string, map[string]string) (int64, string, error),
) (map[string]string, string, string, string, string, string, string, string, string) {
token := make(map[string]string)
var remoteType, remoteRef, remoteHost, remoteUsername, remoteRepo string
Expand Down Expand Up @@ -198,7 +205,7 @@ func ProcessDescriptionURL(descriptionURL string,
// It returns a list of structures representing: the contents of DESCRIPTION file
// for the packages and various information about git repositories storing the packages.
func DownloadDescriptionFiles(packageDescriptionList []string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) []DescriptionFile {
downloadFileFunction func(string, map[string]string) (int64, string, error)) []DescriptionFile {
var inputDescriptionFiles []DescriptionFile
for _, packageDescriptionURL := range packageDescriptionList {
token, remoteType, packageSource, remoteHost, remoteUsername, remoteRepo, remoteSubdir, remoteRef, remoteSha :=
Expand All @@ -208,8 +215,8 @@ func DownloadDescriptionFiles(packageDescriptionList []string,
", remoteUsername = ", remoteUsername, ", remoteRepo = ", remoteRepo,
", remoteSubdir = ", remoteSubdir, ", remoteRef = ", remoteRef, ", remoteSha = ", remoteSha,
)
statusCode, _, descriptionContent := downloadFileFunction(packageDescriptionURL, token)
if statusCode == 200 {
_, descriptionContent, err := downloadFileFunction(packageDescriptionURL, token)
if err == nil {
inputDescriptionFiles = append(
inputDescriptionFiles,
DescriptionFile{
Expand All @@ -231,7 +238,7 @@ func DownloadDescriptionFiles(packageDescriptionList []string,
// GetPackagesFileContent downloads the PACKAGES file from the repositoryURL using the downloadFileFunction
// and returns the contents, or empty string in case of error.
func GetPackagesFileContent(repositoryURL string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) string {
downloadFileFunction func(string, map[string]string) (int64, string, error)) string {
var packagesFileURL string
if strings.Contains(repositoryURL, "/bin/windows/") || strings.Contains(repositoryURL, "/bin/macosx") {
// If we're dealing with a repository with binary Windows or macOS packages,
Expand All @@ -241,8 +248,8 @@ func GetPackagesFileContent(repositoryURL string,
packagesFileURL = repositoryURL + "/src/contrib/PACKAGES"
}
log.Debug("Downloading ", packagesFileURL)
statusCode, _, packagesFileContent := downloadFileFunction(packagesFileURL, map[string]string{})
if statusCode == 200 {
_, packagesFileContent, err := downloadFileFunction(packagesFileURL, map[string]string{})
if err == nil {
return packagesFileContent
}
log.Warn("An error occurred while downloading ", packagesFileURL)
Expand All @@ -253,7 +260,7 @@ func GetPackagesFileContent(repositoryURL string,
// Returns a map from repository URL to the string with the contents of PACKAGES file
// for that repository.
func DownloadPackagesFiles(repositoryList []string,
downloadFileFunction func(string, map[string]string) (int, int64, string)) map[string]string {
downloadFileFunction func(string, map[string]string) (int64, string, error)) map[string]string {
inputPackagesFiles := make(map[string]string)
for _, repository := range repositoryList {
inputPackagesFiles[repository] = GetPackagesFileContent(repository, downloadFileFunction)
Expand Down
97 changes: 49 additions & 48 deletions cmd/download_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,152 +17,153 @@ limitations under the License.
package cmd

import (
"errors"
"testing"

"github.com/stretchr/testify/assert"
)

func mockedDownloadTextFile(url string, _ map[string]string) (int, int64, string) { // nolint: gocyclo
func mockedDownloadTextFile(url string, _ map[string]string) (int64, string, error) { // nolint: gocyclo
switch {
case url == "https://gitlab.example.com/api/v4/projects/37706/repository/tags/v1.3.1":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "aaabbbcccddd111"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/37706":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group1/group2/project1"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/38706/repository/tags/v1.4.2":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "aaa222ccc444111"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/38706":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group3/group4/group5/project4"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/30176/repository/tags/v0.2.0":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "fff222ccc444eee"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/30176":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group6/project7"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39307/repository/branches/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "fff555ddd888eee"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39307":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group7/project8"
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39211/repository/branches/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"commit": {
"someotherdata": "someotherdata",
"id": "fffeee999888aaa"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/39211":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"path_with_namespace": "group9/subgroup10/subgroup11/project9"
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/formatters/git/ref/tags/v0.5.4":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "444eee222111eee"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/rtables/git/ref/tags/v0.6.5":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "555ddd222111ddd"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/nestcolor/git/ref/heads/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "555333aaabbbddd"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/tern/git/ref/heads/main":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "555333aaaeeefff"
}
}`
}`, nil
case url == "https://api.github.com/repos/insightsengineering/rlistings/git/ref/tags/v0.2.6":
return 200, 0, `{
return 0, `{
"someotherdata": "someotherdata",
"object": {
"someotherdata": "someotherdata",
"sha": "111444999eee222"
}
}`
}`, nil
case url == "https://gitlab.example.com/api/v4/projects/37706/repository/files/subdirectory%2FDESCRIPTION/raw?ref=v1.3.1":
return 200, 0, "DESCRIPTION contents 1"
return 0, "DESCRIPTION contents 1", nil
case url == "https://gitlab.example.com/api/v4/projects/38706/repository/files/subdirectory1%2Fsubdirectory2%2FDESCRIPTION/raw?ref=v1.4.2":
return 200, 0, "DESCRIPTION contents 2"
return 0, "DESCRIPTION contents 2", nil
case url == "https://gitlab.example.com/api/v4/projects/30176/repository/files/DESCRIPTION/raw?ref=v0.2.0":
return 200, 0, "DESCRIPTION contents 3"
return 0, "DESCRIPTION contents 3", nil
case url == "https://gitlab.example.com/api/v4/projects/39307/repository/files/DESCRIPTION/raw?ref=main":
return 200, 0, "DESCRIPTION contents 4"
return 0, "DESCRIPTION contents 4", nil
case url == "https://gitlab.example.com/api/v4/projects/39211/repository/files/subdirectory1%2FDESCRIPTION/raw?ref=main":
return 200, 0, "DESCRIPTION contents 5"
return 0, "DESCRIPTION contents 5", nil
case url == "https://raw.githubusercontent.com/insightsengineering/formatters/v0.5.4/subdirectory/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 6"
return 0, "DESCRIPTION contents 6", nil
case url == "https://raw.githubusercontent.com/insightsengineering/rtables/v0.6.5/subdirectory1/subdirectory2/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 7"
return 0, "DESCRIPTION contents 7", nil
case url == "https://raw.githubusercontent.com/insightsengineering/nestcolor/main/subdirectory/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 8"
return 0, "DESCRIPTION contents 8", nil
case url == "https://raw.githubusercontent.com/insightsengineering/tern/main/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 9"
return 0, "DESCRIPTION contents 9", nil
case url == "https://raw.githubusercontent.com/insightsengineering/rlistings/v0.2.6/DESCRIPTION":
return 200, 0, "DESCRIPTION contents 10"
return 0, "DESCRIPTION contents 10", nil
case url == "https://repo1.example.com/repo1/src/contrib/PACKAGES":
return 200, 0, "PACKAGES contents 1"
return 0, "PACKAGES contents 1", nil
case url == "https://repo2.example.com/repo2/src/contrib/PACKAGES":
return 200, 0, "PACKAGES contents 2"
return 0, "PACKAGES contents 2", nil
case url == "https://repo3.example.com/repo3/src/contrib/PACKAGES":
return 200, 0, "PACKAGES contents 3"
return 0, "PACKAGES contents 3", nil
case url == "https://cloud.r-project.org/bin/windows/contrib/4.3/PACKAGES":
return 200, 0, "PACKAGES content bin/windows"
return 0, "PACKAGES content bin/windows", nil
case url == "https://cloud.r-project.org/src/contrib/PACKAGES":
return 200, 0, "PACKAGES content src/contrib"
return 0, "PACKAGES content src/contrib", nil
case url == "https://example.com/src/contrib/PACKAGES":
return 404, 0, "Not found"
return 0, "", errors.New("Not found")
}
return 200, 0, ""
return 0, "", nil
}

func Test_DownloadDescriptionFiles(t *testing.T) {
Expand Down

0 comments on commit 6a26580

Please sign in to comment.