Skip to content

Commit

Permalink
fix(cmd): prevent sudden crash on download error (#1432)
Browse files Browse the repository at this point in the history
  • Loading branch information
b00f authored Jul 25, 2024
1 parent 43dcd1b commit 52ceb19
Show file tree
Hide file tree
Showing 5 changed files with 108 additions and 70 deletions.
7 changes: 2 additions & 5 deletions cmd/daemon/import.go
Original file line number Diff line number Diff line change
Expand Up @@ -81,11 +81,8 @@ func buildImportCmd(parentCmd *cobra.Command) {

cmd.PrintLine()

importer.Download(
c.Context(),
&selected,
downloadProgressBar,
)
err = importer.Download(c.Context(), &selected, downloadProgressBar)
cmd.FatalErrorCheck(err)

cmd.PrintLine()
cmd.PrintLine()
Expand Down
10 changes: 4 additions & 6 deletions cmd/gtk/startup_assistant.go
Original file line number Diff line number Diff line change
Expand Up @@ -258,12 +258,8 @@ func startupAssistant(workingDir string, chainType genesis.ChainType) bool {
go func() {
log.Printf("start downloading...\n")

importer.Download(
ctx,
&md[snapshotIndex],
func(fileName string, totalSize, downloaded int64,
percentage float64,
) {
err := importer.Download(ctx, &md[snapshotIndex],
func(fileName string, totalSize, downloaded int64, percentage float64) {
percent := int(percentage)
glib.IdleAdd(func() {
dlMessage := fmt.Sprintf("🌐 Downloading %s | %d%% (%s / %s)",
Expand All @@ -278,6 +274,8 @@ func startupAssistant(workingDir string, chainType genesis.ChainType) bool {
)

glib.IdleAdd(func() {
fatalErrorCheck(err)

log.Printf("extracting data...\n")
ssPBLabel.SetText(" " + "📂 Extracting downloaded files...")
err := importer.ExtractAndStoreFiles()
Expand Down
39 changes: 19 additions & 20 deletions cmd/importer.go
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import (
"errors"
"fmt"
"io"
"log"
"net/http"
"net/url"
"os"
Expand All @@ -23,7 +24,7 @@ const DefaultSnapshotURL = "https://snapshot.pactus.org"

const maxDecompressedSize = 10 << 20 // 10 MB

type DMStateFunc func(
type ImporterStateFunc func(
fileName string,
totalSize, downloaded int64,
percentage float64,
Expand Down Expand Up @@ -124,47 +125,45 @@ func (i *Importer) GetMetadata(ctx context.Context) ([]Metadata, error) {
return metadata, nil
}

func (i *Importer) Download(
ctx context.Context,
metadata *Metadata,
stateFunc DMStateFunc,
) {
done := make(chan struct{})
func (i *Importer) Download(ctx context.Context, metadata *Metadata,
stateFunc ImporterStateFunc,
) error {
done := make(chan error)
defer close((done))

dlLink, err := url.JoinPath(i.snapshotURL, metadata.Data.Path)
FatalErrorCheck(err)
if err != nil {
return err
}

fileName := filepath.Base(dlLink)

i.dataFileName = fileName

filePath := fmt.Sprintf("%s/%s", i.tempDir, fileName)

d := downloader.New(
dlLink,
filePath,
metadata.Data.Sha,
)

d := downloader.New(dlLink, filePath, metadata.Data.Sha)
d.Start(ctx)

go func() {
err := <-d.Errors()
FatalErrorCheck(err)
if err != nil {
log.Printf("download encountered an error: %s\n", err)
done <- err
}
}()

go func() {
for state := range d.Stats() {
stateFunc(fileName, state.TotalSize, state.Downloaded, state.Percent)
if state.Completed {
done <- struct{}{}
close(done)
log.Println("download completed")
done <- nil

return
}
}
}()

<-done
return <-done
}

func (i *Importer) Cleanup() error {
Expand Down
12 changes: 9 additions & 3 deletions util/io.go
Original file line number Diff line number Diff line change
Expand Up @@ -90,8 +90,7 @@ func IsDirEmpty(name string) bool {
return errors.Is(err, io.EOF)
}

// IsDirNotExistsOrEmpty returns true if a directory does not exist or is empty.
// It checks if the path exists and, if so, whether the directory is empty.
// IsDirNotExistsOrEmpty checks if the path exists and, if so, whether the directory is empty.
func IsDirNotExistsOrEmpty(name string) bool {
if !PathExists(name) {
return true
Expand Down Expand Up @@ -203,11 +202,18 @@ func NewFixedReader(max int, buf []byte) *FixedReader {

// MoveDirectory moves a directory from srcDir to dstDir, including all its contents.
// If dstDir already exists and is not empty, it returns an error.
// If the parent directory of dstDir does not exist, it will be created.
func MoveDirectory(srcDir, dstDir string) error {
if !IsDirNotExistsOrEmpty(dstDir) {
if PathExists(dstDir) {
return fmt.Errorf("destination directory %s already exists", dstDir)
}

// Get the parent directory of the destination directory
parentDir := filepath.Dir(dstDir)
if err := Mkdir(parentDir); err != nil {
return fmt.Errorf("failed to create parent directories for %s: %w", dstDir, err)
}

if err := os.Rename(srcDir, dstDir); err != nil {
return fmt.Errorf("failed to move directory from %s to %s: %w", srcDir, dstDir, err)
}
Expand Down
110 changes: 74 additions & 36 deletions util/io_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,8 @@ import (
"github.com/stretchr/testify/assert"
)

const invalidDirName = "/invalid:path/\x00*folder?\\CON"

func TestWriteFile(t *testing.T) {
p := TempDirPath()
d := []byte("some-data")
Expand Down Expand Up @@ -90,56 +92,92 @@ func TestIsValidPath(t *testing.T) {
assert.False(t, IsValidDirPath("/root"))
assert.False(t, IsValidDirPath("/test"))
}
assert.False(t, IsValidDirPath(invalidDirName))
assert.False(t, IsValidDirPath("./io_test.go"))
assert.True(t, IsValidDirPath("/tmp"))
assert.True(t, IsValidDirPath("/tmp/pactus"))
}

func TestMoveDirectory(t *testing.T) {
// Create temporary directories
srcDir := TempDirPath()
dstDir := TempDirPath()
defer func() { _ = os.RemoveAll(srcDir) }()
defer func() { _ = os.RemoveAll(dstDir) }()

// Create a subdirectory in the source directory
subDir := filepath.Join(srcDir, "subdir")
err := Mkdir(subDir)
assert.NoError(t, err)
t.Run("DestinationDirectoryExistsAndNotEmpty", func(t *testing.T) {
srcDir := TempDirPath()
dstDir := TempDirPath()

// Create multiple files in the subdirectory
files := []struct {
name string
content string
}{
{"file1.txt", "content 1"},
{"file2.txt", "content 2"},
}
err := MoveDirectory(srcDir, dstDir)
assert.Error(t, err)
assert.Contains(t, err.Error(), "destination directory")
})

for _, file := range files {
filePath := filepath.Join(subDir, file.name)
err = WriteFile(filePath, []byte(file.content))
t.Run("ParentDirectoryCreationFailure", func(t *testing.T) {
srcDir := TempDirPath()

err := MoveDirectory(srcDir, invalidDirName)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to create parent directories")
})

t.Run("SourceDirectoryRenameFailure", func(t *testing.T) {
srcDir := TempDirPath()
dstDir := TempDirPath()

err := os.RemoveAll(dstDir)
assert.NoError(t, err)
}

// Move the directory
dstDirPath := filepath.Join(dstDir, "movedir")
err = MoveDirectory(srcDir, dstDirPath)
assert.NoError(t, err)
// Remove the source directory to simulate the rename failure
err = os.RemoveAll(srcDir)
assert.NoError(t, err)

// Assert the source directory no longer exists
assert.False(t, PathExists(srcDir))
err = MoveDirectory(srcDir, dstDir)
assert.Error(t, err)
assert.Contains(t, err.Error(), "failed to move directory")
})

// Assert the destination directory exists
assert.True(t, PathExists(dstDirPath))
t.Run("MoveDirectorySuccess", func(t *testing.T) {
// Create temporary directories
srcDir := TempDirPath()
dstDir := TempDirPath()
defer func() { _ = os.RemoveAll(srcDir) }()
defer func() { _ = os.RemoveAll(dstDir) }()

// Verify that all files have been moved and their contents are correct
for _, file := range files {
movedFilePath := filepath.Join(dstDirPath, "subdir", file.name)
data, err := ReadFile(movedFilePath)
// Create a subdirectory in the source directory
subDir := filepath.Join(srcDir, "subdir")
err := Mkdir(subDir)
assert.NoError(t, err)
assert.Equal(t, file.content, string(data))
}

// Create multiple files in the subdirectory
files := []struct {
name string
content string
}{
{"file1.txt", "content 1"},
{"file2.txt", "content 2"},
}

for _, file := range files {
filePath := filepath.Join(subDir, file.name)
err = WriteFile(filePath, []byte(file.content))
assert.NoError(t, err)
}

// Move the directory
dstDirPath := filepath.Join(dstDir, "movedir")
err = MoveDirectory(srcDir, dstDirPath)
assert.NoError(t, err)

// Assert the source directory no longer exists
assert.False(t, PathExists(srcDir))

// Assert the destination directory exists
assert.True(t, PathExists(dstDirPath))

// Verify that all files have been moved and their contents are correct
for _, file := range files {
movedFilePath := filepath.Join(dstDirPath, "subdir", file.name)
data, err := ReadFile(movedFilePath)
assert.NoError(t, err)
assert.Equal(t, file.content, string(data))
}
})
}

func TestSanitizeArchivePath(t *testing.T) {
Expand Down

0 comments on commit 52ceb19

Please sign in to comment.