Skip to content

Commit

Permalink
add config structs json/yaml conversion support
Browse files Browse the repository at this point in the history
  • Loading branch information
iychoi committed Oct 1, 2024
1 parent 57d7ea1 commit 940a9ea
Show file tree
Hide file tree
Showing 5 changed files with 182 additions and 50 deletions.
45 changes: 24 additions & 21 deletions fs/cache.go
Original file line number Diff line number Diff line change
Expand Up @@ -12,30 +12,30 @@ import (

// MetadataCacheTimeoutSetting defines cache timeout for path
type MetadataCacheTimeoutSetting struct {
Path string
Timeout time.Duration
Inherit bool
Path string `yaml:"path" json:"path"`
Timeout types.Duration `yaml:"timeout" json:"timeout"`
Inherit bool `yaml:"inherit,omitempty" json:"inherit,omitempty"`
}

// CacheConfig defines cache config
type CacheConfig struct {
Timeout time.Duration // cache timeout
CleanupTime time.Duration //
MetadataTimeoutSettings []MetadataCacheTimeoutSetting
Timeout types.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"` // cache timeout
CleanupTime types.Duration `yaml:"cleanup_time,omitempty" json:"cleanup_time,omitempty"` // cache cleanup time
MetadataTimeoutSettings []MetadataCacheTimeoutSetting `yaml:"metadata_timeout_settings,omitempty" json:"metadata_timeout_settings,omitempty"`
// determine if we will invalidate parent dir's entry cache
// at subdir/file creation/deletion
// turn to false to allow short cache inconsistency
InvalidateParentEntryCacheImmediately bool
InvalidateParentEntryCacheImmediately bool `yaml:"invalidate_parent_entry_cache_immediately,omitempty" json:"invalidate_parent_entry_cache_immediately,omitempty"`
// for mysql iCAT backend, this should be true.
// for postgresql iCAT backend, this can be false.
StartNewTransaction bool
StartNewTransaction bool `yaml:"start_new_transaction,omitempty" json:"start_new_transaction,omitempty"`
}

// NewDefaultCacheConfig creates a new default CacheConfig
func NewDefaultCacheConfig() CacheConfig {
return CacheConfig{
Timeout: FileSystemTimeoutDefault,
CleanupTime: FileSystemTimeoutDefault,
Timeout: types.Duration(FileSystemTimeoutDefault),
CleanupTime: types.Duration(FileSystemTimeoutDefault),
MetadataTimeoutSettings: []MetadataCacheTimeoutSetting{},
InvalidateParentEntryCacheImmediately: true,
StartNewTransaction: true,
Expand All @@ -61,15 +61,18 @@ type FileSystemCache struct {

// NewFileSystemCache creates a new FileSystemCache
func NewFileSystemCache(config *CacheConfig) *FileSystemCache {
entryCache := gocache.New(config.Timeout, config.CleanupTime)
negativeEntryCache := gocache.New(config.Timeout, config.CleanupTime)
dirCache := gocache.New(config.Timeout, config.CleanupTime)
metadataCache := gocache.New(config.Timeout, config.CleanupTime)
groupUsersCache := gocache.New(config.Timeout, config.CleanupTime)
userGroupsCache := gocache.New(config.Timeout, config.CleanupTime)
groupsCache := gocache.New(config.Timeout, config.CleanupTime)
usersCache := gocache.New(config.Timeout, config.CleanupTime)
aclCache := gocache.New(config.Timeout, config.CleanupTime)
timeout := time.Duration(config.Timeout)
cleanupTime := time.Duration(config.CleanupTime)

entryCache := gocache.New(timeout, cleanupTime)
negativeEntryCache := gocache.New(timeout, cleanupTime)
dirCache := gocache.New(timeout, cleanupTime)
metadataCache := gocache.New(timeout, cleanupTime)
groupUsersCache := gocache.New(timeout, cleanupTime)
userGroupsCache := gocache.New(timeout, cleanupTime)
groupsCache := gocache.New(timeout, cleanupTime)
usersCache := gocache.New(timeout, cleanupTime)
aclCache := gocache.New(timeout, cleanupTime)

// build a map for quick search
cacheTimeoutSettingMap := map[string]MetadataCacheTimeoutSetting{}
Expand Down Expand Up @@ -103,7 +106,7 @@ func (cache *FileSystemCache) getCacheTTLForPath(path string) time.Duration {
// check map first
if timeoutSetting, ok := cache.cacheTimeoutPathMap[path]; ok {
// exact match
return timeoutSetting.Timeout
return time.Duration(timeoutSetting.Timeout)
}

// check inherit
Expand All @@ -115,7 +118,7 @@ func (cache *FileSystemCache) getCacheTTLForPath(path string) time.Duration {
// parent match
if timeoutSetting.Inherit {
// inherit
return timeoutSetting.Timeout
return time.Duration(timeoutSetting.Timeout)
}
}
}
Expand Down
57 changes: 29 additions & 28 deletions fs/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@ import (
"time"

"github.com/cyverse/go-irodsclient/irods/session"
"github.com/cyverse/go-irodsclient/irods/types"
)

const (
Expand Down Expand Up @@ -37,52 +38,52 @@ const (

// ConnectionConfig is a struct that stores configuration for connections
type ConnectionConfig struct {
CreationTimeout time.Duration // timeout for creating a new connection
InitNumber int // number of connections created when init
MaxNumber int // max number of connections
MaxIdleNumber int // max number of idle connections
Lifespan time.Duration // connection's lifespan (max time to be reused)
OperationTimeout time.Duration // timeout for iRODS operations
IdleTimeout time.Duration // time out for being idle, after this point the connection will be disposed
TCPBufferSize int // buffer size
CreationTimeout types.Duration `yaml:"creation_timeout,omitempty" json:"creation_timeout,omitempty"` // timeout for creating a new connection
InitNumber int `yaml:"init_number,omitempty" json:"init_number,omitempty"` // number of connections created when init
MaxNumber int `yaml:"max_number,omitempty" json:"max_number,omitempty"` // max number of connections
MaxIdleNumber int `yaml:"max_idle_number,omitempty" json:"max_idle_number,omitempty"` // max number of idle connections
Lifespan types.Duration `yaml:"lifespan,omitempty" json:"lifespan,omitempty"` // connection's lifespan (max time to be reused)
OperationTimeout types.Duration `yaml:"operation_timeout,omitempty" json:"operation_timeout,omitempty"` // timeout for iRODS operations
IdleTimeout types.Duration `yaml:"idle_timeout,omitempty" json:"idle_timeout,omitempty"` // time out for being idle, after this point the connection will be disposed
TCPBufferSize int `yaml:"tcp_buffer_size,omitempty" json:"tcp_buffer_size,omitempty"` // buffer size
}

// NewDefaultMetadataConnectionConfig creates a default ConnectionConfig for metadata
func NewDefaultMetadataConnectionConfig() ConnectionConfig {
return ConnectionConfig{
CreationTimeout: FileSystemConnectionCreationTimeoutDefault,
CreationTimeout: types.Duration(FileSystemConnectionCreationTimeoutDefault),
InitNumber: FileSystemMetadataConnectionInitNumberDefault,
MaxNumber: FileSystemMetadataConnectionMaxNumberDefault,
MaxIdleNumber: FileSystemMetadataConnectionMaxIdleNumberDefault,
Lifespan: FileSystemConnectionLifespanDefault,
OperationTimeout: FileSystemTimeoutDefault,
IdleTimeout: FileSystemTimeoutDefault,
Lifespan: types.Duration(FileSystemConnectionLifespanDefault),
OperationTimeout: types.Duration(FileSystemTimeoutDefault),
IdleTimeout: types.Duration(FileSystemTimeoutDefault),
TCPBufferSize: FileSystemTCPBufferSizeDefault,
}
}

// NewDefaultIOConnectionConfig creates a default ConnectionConfig for IO
func NewDefaultIOConnectionConfig() ConnectionConfig {
return ConnectionConfig{
CreationTimeout: FileSystemConnectionCreationTimeoutDefault,
CreationTimeout: types.Duration(FileSystemConnectionCreationTimeoutDefault),
InitNumber: FileSystemIOConnectionInitNumberDefault,
MaxNumber: FileSystemIOConnectionMaxNumberDefault,
MaxIdleNumber: FileSystemIOConnectionMaxIdleNumberDefault,
Lifespan: FileSystemConnectionLifespanDefault,
OperationTimeout: FileSystemTimeoutDefault,
IdleTimeout: FileSystemTimeoutDefault,
Lifespan: types.Duration(FileSystemConnectionLifespanDefault),
OperationTimeout: types.Duration(FileSystemTimeoutDefault),
IdleTimeout: types.Duration(FileSystemTimeoutDefault),
TCPBufferSize: FileSystemTCPBufferSizeDefault,
}
}

// FileSystemConfig is a struct for file system configuration
type FileSystemConfig struct {
ApplicationName string
ApplicationName string `yaml:"application_name,omitempty" json:"application_name,omitempty"`

MetadataConnection ConnectionConfig
IOConnection ConnectionConfig
MetadataConnection ConnectionConfig `yaml:"metadata_connection,omitempty" json:"metadata_connection,omitempty"`
IOConnection ConnectionConfig `yaml:"io_connection,omitempty" json:"io_connection,omitempty"`

Cache CacheConfig
Cache CacheConfig `yaml:"cache,omitempty" json:"cache,omitempty"`

AddressResolver session.AddressResolver
}
Expand All @@ -104,11 +105,11 @@ func NewFileSystemConfig(applicationName string) *FileSystemConfig {
func (config *FileSystemConfig) ToMetadataSessionConfig() *session.IRODSSessionConfig {
sessionConfig := session.NewIRODSSessionConfig(config.ApplicationName)

sessionConfig.ConnectionCreationTimeout = config.MetadataConnection.CreationTimeout
sessionConfig.ConnectionCreationTimeout = time.Duration(config.MetadataConnection.CreationTimeout)
sessionConfig.ConnectionInitNumber = config.MetadataConnection.InitNumber
sessionConfig.ConnectionLifespan = config.MetadataConnection.Lifespan
sessionConfig.OperationTimeout = config.MetadataConnection.OperationTimeout
sessionConfig.ConnectionIdleTimeout = config.MetadataConnection.IdleTimeout
sessionConfig.ConnectionLifespan = time.Duration(config.MetadataConnection.Lifespan)
sessionConfig.OperationTimeout = time.Duration(config.MetadataConnection.OperationTimeout)
sessionConfig.ConnectionIdleTimeout = time.Duration(config.MetadataConnection.IdleTimeout)
sessionConfig.ConnectionMaxNumber = config.MetadataConnection.MaxNumber
sessionConfig.ConnectionMaxIdleNumber = config.MetadataConnection.MaxIdleNumber
sessionConfig.TCPBufferSize = config.MetadataConnection.TCPBufferSize
Expand All @@ -123,11 +124,11 @@ func (config *FileSystemConfig) ToMetadataSessionConfig() *session.IRODSSessionC
func (config *FileSystemConfig) ToIOSessionConfig() *session.IRODSSessionConfig {
sessionConfig := session.NewIRODSSessionConfig(config.ApplicationName)

sessionConfig.ConnectionCreationTimeout = config.IOConnection.CreationTimeout
sessionConfig.ConnectionCreationTimeout = time.Duration(config.IOConnection.CreationTimeout)
sessionConfig.ConnectionInitNumber = config.IOConnection.InitNumber
sessionConfig.ConnectionLifespan = config.IOConnection.Lifespan
sessionConfig.OperationTimeout = config.IOConnection.OperationTimeout
sessionConfig.ConnectionIdleTimeout = config.IOConnection.IdleTimeout
sessionConfig.ConnectionLifespan = time.Duration(config.IOConnection.Lifespan)
sessionConfig.OperationTimeout = time.Duration(config.IOConnection.OperationTimeout)
sessionConfig.ConnectionIdleTimeout = time.Duration(config.IOConnection.IdleTimeout)
sessionConfig.ConnectionMaxNumber = config.IOConnection.MaxNumber
sessionConfig.ConnectionMaxIdleNumber = config.IOConnection.MaxIdleNumber
sessionConfig.TCPBufferSize = config.IOConnection.TCPBufferSize
Expand Down
2 changes: 1 addition & 1 deletion fs/fs.go
Original file line number Diff line number Diff line change
Expand Up @@ -308,7 +308,7 @@ func (fs *FileSystem) RemoveFile(path string, force bool) error {

defer fs.fileHandleMap.RemoveCloseEventHandler(eventHandlerID)

if util.WaitTimeout(&wg, fs.config.MetadataConnection.OperationTimeout) {
if util.WaitTimeout(&wg, time.Duration(fs.config.MetadataConnection.OperationTimeout)) {
// timed out
return xerrors.Errorf("failed to remove file, there are files still opened")
}
Expand Down
67 changes: 67 additions & 0 deletions irods/types/duration.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
package types

import (
"encoding/json"
"fmt"
"time"

"golang.org/x/xerrors"
)

// Duration is a replacement of time.Duration that supports JSON
type Duration time.Duration

// MarshalJSON ...
func (d *Duration) MarshalJSON() ([]byte, error) {
return json.Marshal(time.Duration(*d).String())
}

// UnmarshalJSON ...
func (d *Duration) UnmarshalJSON(b []byte) error {
var v interface{}
if err := json.Unmarshal(b, &v); err != nil {
return xerrors.Errorf("failed to parse '%s' to time.Duration: %w", string(b), err)
}
switch value := v.(type) {
case float64:
*d = Duration(time.Duration(value))
return nil
case string:
tmp, err := time.ParseDuration(value)
if err != nil {
return xerrors.Errorf("failed to parse '%s' to time.Duration: %w", string(b), err)
}
*d = Duration(tmp)
return nil
default:
return xerrors.Errorf("failed to parse '%s' to time.Duration", string(b))
}
}

// bug in YAMLv2, fixed in YAMLv3
// // MarshalYAML ...
// func (d *Duration) MarshalYAML() (interface{}, error) {
// return time.Duration(*d).String(), nil
// }

// UnmarshalYAML ...
func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error {
var tm string
if err := unmarshal(&tm); err != nil {
return xerrors.Errorf("failed to parse '%s' to time.Duration: %w", tm, err)
}

lastChar := byte(tm[len(tm)-1])
if lastChar >= '0' && lastChar <= '9' {
// ends with number, no units
tm = tm + "ns"
}

td, err := time.ParseDuration(tm)
if err != nil {
return fmt.Errorf("failed to parse '%s' to time.Duration: %w", tm, err)
}

*d = Duration(td)
return nil
}
61 changes: 61 additions & 0 deletions test/testcases/duration_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,61 @@
package testcases

import (
"testing"
"time"

"github.com/cyverse/go-irodsclient/irods/types"
"github.com/stretchr/testify/assert"
"gopkg.in/yaml.v2"
)

func TestDuration(t *testing.T) {
t.Run("test YAMLMarshal", testYAMLMarshal)
t.Run("test YAMLUnmarshalFromStringWithoutUnits", testYAMLUnmarshalFromStringWithoutUnits)
t.Run("test YAMLUnmarshalFromString", testYAMLUnmarshalFromString)
}

func testYAMLMarshal(t *testing.T) {
d1 := types.Duration(5 * time.Minute)

yamlBytes, err := yaml.Marshal(d1)
assert.NoError(t, err)
assert.NotEmpty(t, string(yamlBytes))

var d2 types.Duration
err = yaml.Unmarshal(yamlBytes, &d2)
assert.NoError(t, err)
assert.Equal(t, d1, d2)
}

func testYAMLUnmarshalFromStringWithoutUnits(t *testing.T) {
v1 := []byte("60000000000")

var d1 types.Duration
err := yaml.Unmarshal(v1, &d1)
assert.NoError(t, err)
assert.Equal(t, 1*time.Minute, time.Duration(d1))

v2 := []byte("6h60000000000")

var d2 types.Duration
err = yaml.Unmarshal(v2, &d2)
assert.NoError(t, err)
assert.Equal(t, 6*time.Hour+1*time.Minute, time.Duration(d2))
}

func testYAMLUnmarshalFromString(t *testing.T) {
v1 := []byte("6m")

var d1 types.Duration
err := yaml.Unmarshal(v1, &d1)
assert.NoError(t, err)
assert.Equal(t, 6*time.Minute, time.Duration(d1))

v2 := []byte("6h6m")

var d2 types.Duration
err = yaml.Unmarshal(v2, &d2)
assert.NoError(t, err)
assert.Equal(t, 6*time.Hour+6*time.Minute, time.Duration(d2))
}

0 comments on commit 940a9ea

Please sign in to comment.