diff --git a/fs/cache.go b/fs/cache.go index 48c9668..c27a2f5 100644 --- a/fs/cache.go +++ b/fs/cache.go @@ -12,30 +12,30 @@ import ( // MetadataCacheTimeoutSetting defines cache timeout for path type MetadataCacheTimeoutSetting struct { - Path string - Timeout time.Duration - Inherit bool + Path string `yaml:"path" json:"path"` + Timeout types.Duration `yaml:"timeout" json:"timeout"` + Inherit bool `yaml:"inherit,omitempty" json:"inherit,omitempty"` } // CacheConfig defines cache config type CacheConfig struct { - Timeout time.Duration // cache timeout - CleanupTime time.Duration // - MetadataTimeoutSettings []MetadataCacheTimeoutSetting + Timeout types.Duration `yaml:"timeout,omitempty" json:"timeout,omitempty"` // cache timeout + CleanupTime types.Duration `yaml:"cleanup_time,omitempty" json:"cleanup_time,omitempty"` // cache cleanup time + MetadataTimeoutSettings []MetadataCacheTimeoutSetting `yaml:"metadata_timeout_settings,omitempty" json:"metadata_timeout_settings,omitempty"` // determine if we will invalidate parent dir's entry cache // at subdir/file creation/deletion // turn to false to allow short cache inconsistency - InvalidateParentEntryCacheImmediately bool + InvalidateParentEntryCacheImmediately bool `yaml:"invalidate_parent_entry_cache_immediately,omitempty" json:"invalidate_parent_entry_cache_immediately,omitempty"` // for mysql iCAT backend, this should be true. // for postgresql iCAT backend, this can be false. - StartNewTransaction bool + StartNewTransaction bool `yaml:"start_new_transaction,omitempty" json:"start_new_transaction,omitempty"` } // NewDefaultCacheConfig creates a new default CacheConfig func NewDefaultCacheConfig() CacheConfig { return CacheConfig{ - Timeout: FileSystemTimeoutDefault, - CleanupTime: FileSystemTimeoutDefault, + Timeout: types.Duration(FileSystemTimeoutDefault), + CleanupTime: types.Duration(FileSystemTimeoutDefault), MetadataTimeoutSettings: []MetadataCacheTimeoutSetting{}, InvalidateParentEntryCacheImmediately: true, StartNewTransaction: true, @@ -61,15 +61,18 @@ type FileSystemCache struct { // NewFileSystemCache creates a new FileSystemCache func NewFileSystemCache(config *CacheConfig) *FileSystemCache { - entryCache := gocache.New(config.Timeout, config.CleanupTime) - negativeEntryCache := gocache.New(config.Timeout, config.CleanupTime) - dirCache := gocache.New(config.Timeout, config.CleanupTime) - metadataCache := gocache.New(config.Timeout, config.CleanupTime) - groupUsersCache := gocache.New(config.Timeout, config.CleanupTime) - userGroupsCache := gocache.New(config.Timeout, config.CleanupTime) - groupsCache := gocache.New(config.Timeout, config.CleanupTime) - usersCache := gocache.New(config.Timeout, config.CleanupTime) - aclCache := gocache.New(config.Timeout, config.CleanupTime) + timeout := time.Duration(config.Timeout) + cleanupTime := time.Duration(config.CleanupTime) + + entryCache := gocache.New(timeout, cleanupTime) + negativeEntryCache := gocache.New(timeout, cleanupTime) + dirCache := gocache.New(timeout, cleanupTime) + metadataCache := gocache.New(timeout, cleanupTime) + groupUsersCache := gocache.New(timeout, cleanupTime) + userGroupsCache := gocache.New(timeout, cleanupTime) + groupsCache := gocache.New(timeout, cleanupTime) + usersCache := gocache.New(timeout, cleanupTime) + aclCache := gocache.New(timeout, cleanupTime) // build a map for quick search cacheTimeoutSettingMap := map[string]MetadataCacheTimeoutSetting{} @@ -103,7 +106,7 @@ func (cache *FileSystemCache) getCacheTTLForPath(path string) time.Duration { // check map first if timeoutSetting, ok := cache.cacheTimeoutPathMap[path]; ok { // exact match - return timeoutSetting.Timeout + return time.Duration(timeoutSetting.Timeout) } // check inherit @@ -115,7 +118,7 @@ func (cache *FileSystemCache) getCacheTTLForPath(path string) time.Duration { // parent match if timeoutSetting.Inherit { // inherit - return timeoutSetting.Timeout + return time.Duration(timeoutSetting.Timeout) } } } diff --git a/fs/config.go b/fs/config.go index ad6c07e..444e083 100644 --- a/fs/config.go +++ b/fs/config.go @@ -4,6 +4,7 @@ import ( "time" "github.com/cyverse/go-irodsclient/irods/session" + "github.com/cyverse/go-irodsclient/irods/types" ) const ( @@ -37,26 +38,26 @@ const ( // ConnectionConfig is a struct that stores configuration for connections type ConnectionConfig struct { - CreationTimeout time.Duration // timeout for creating a new connection - InitNumber int // number of connections created when init - MaxNumber int // max number of connections - MaxIdleNumber int // max number of idle connections - Lifespan time.Duration // connection's lifespan (max time to be reused) - OperationTimeout time.Duration // timeout for iRODS operations - IdleTimeout time.Duration // time out for being idle, after this point the connection will be disposed - TCPBufferSize int // buffer size + CreationTimeout types.Duration `yaml:"creation_timeout,omitempty" json:"creation_timeout,omitempty"` // timeout for creating a new connection + InitNumber int `yaml:"init_number,omitempty" json:"init_number,omitempty"` // number of connections created when init + MaxNumber int `yaml:"max_number,omitempty" json:"max_number,omitempty"` // max number of connections + MaxIdleNumber int `yaml:"max_idle_number,omitempty" json:"max_idle_number,omitempty"` // max number of idle connections + Lifespan types.Duration `yaml:"lifespan,omitempty" json:"lifespan,omitempty"` // connection's lifespan (max time to be reused) + OperationTimeout types.Duration `yaml:"operation_timeout,omitempty" json:"operation_timeout,omitempty"` // timeout for iRODS operations + IdleTimeout types.Duration `yaml:"idle_timeout,omitempty" json:"idle_timeout,omitempty"` // time out for being idle, after this point the connection will be disposed + TCPBufferSize int `yaml:"tcp_buffer_size,omitempty" json:"tcp_buffer_size,omitempty"` // buffer size } // NewDefaultMetadataConnectionConfig creates a default ConnectionConfig for metadata func NewDefaultMetadataConnectionConfig() ConnectionConfig { return ConnectionConfig{ - CreationTimeout: FileSystemConnectionCreationTimeoutDefault, + CreationTimeout: types.Duration(FileSystemConnectionCreationTimeoutDefault), InitNumber: FileSystemMetadataConnectionInitNumberDefault, MaxNumber: FileSystemMetadataConnectionMaxNumberDefault, MaxIdleNumber: FileSystemMetadataConnectionMaxIdleNumberDefault, - Lifespan: FileSystemConnectionLifespanDefault, - OperationTimeout: FileSystemTimeoutDefault, - IdleTimeout: FileSystemTimeoutDefault, + Lifespan: types.Duration(FileSystemConnectionLifespanDefault), + OperationTimeout: types.Duration(FileSystemTimeoutDefault), + IdleTimeout: types.Duration(FileSystemTimeoutDefault), TCPBufferSize: FileSystemTCPBufferSizeDefault, } } @@ -64,25 +65,25 @@ func NewDefaultMetadataConnectionConfig() ConnectionConfig { // NewDefaultIOConnectionConfig creates a default ConnectionConfig for IO func NewDefaultIOConnectionConfig() ConnectionConfig { return ConnectionConfig{ - CreationTimeout: FileSystemConnectionCreationTimeoutDefault, + CreationTimeout: types.Duration(FileSystemConnectionCreationTimeoutDefault), InitNumber: FileSystemIOConnectionInitNumberDefault, MaxNumber: FileSystemIOConnectionMaxNumberDefault, MaxIdleNumber: FileSystemIOConnectionMaxIdleNumberDefault, - Lifespan: FileSystemConnectionLifespanDefault, - OperationTimeout: FileSystemTimeoutDefault, - IdleTimeout: FileSystemTimeoutDefault, + Lifespan: types.Duration(FileSystemConnectionLifespanDefault), + OperationTimeout: types.Duration(FileSystemTimeoutDefault), + IdleTimeout: types.Duration(FileSystemTimeoutDefault), TCPBufferSize: FileSystemTCPBufferSizeDefault, } } // FileSystemConfig is a struct for file system configuration type FileSystemConfig struct { - ApplicationName string + ApplicationName string `yaml:"application_name,omitempty" json:"application_name,omitempty"` - MetadataConnection ConnectionConfig - IOConnection ConnectionConfig + MetadataConnection ConnectionConfig `yaml:"metadata_connection,omitempty" json:"metadata_connection,omitempty"` + IOConnection ConnectionConfig `yaml:"io_connection,omitempty" json:"io_connection,omitempty"` - Cache CacheConfig + Cache CacheConfig `yaml:"cache,omitempty" json:"cache,omitempty"` AddressResolver session.AddressResolver } @@ -104,11 +105,11 @@ func NewFileSystemConfig(applicationName string) *FileSystemConfig { func (config *FileSystemConfig) ToMetadataSessionConfig() *session.IRODSSessionConfig { sessionConfig := session.NewIRODSSessionConfig(config.ApplicationName) - sessionConfig.ConnectionCreationTimeout = config.MetadataConnection.CreationTimeout + sessionConfig.ConnectionCreationTimeout = time.Duration(config.MetadataConnection.CreationTimeout) sessionConfig.ConnectionInitNumber = config.MetadataConnection.InitNumber - sessionConfig.ConnectionLifespan = config.MetadataConnection.Lifespan - sessionConfig.OperationTimeout = config.MetadataConnection.OperationTimeout - sessionConfig.ConnectionIdleTimeout = config.MetadataConnection.IdleTimeout + sessionConfig.ConnectionLifespan = time.Duration(config.MetadataConnection.Lifespan) + sessionConfig.OperationTimeout = time.Duration(config.MetadataConnection.OperationTimeout) + sessionConfig.ConnectionIdleTimeout = time.Duration(config.MetadataConnection.IdleTimeout) sessionConfig.ConnectionMaxNumber = config.MetadataConnection.MaxNumber sessionConfig.ConnectionMaxIdleNumber = config.MetadataConnection.MaxIdleNumber sessionConfig.TCPBufferSize = config.MetadataConnection.TCPBufferSize @@ -123,11 +124,11 @@ func (config *FileSystemConfig) ToMetadataSessionConfig() *session.IRODSSessionC func (config *FileSystemConfig) ToIOSessionConfig() *session.IRODSSessionConfig { sessionConfig := session.NewIRODSSessionConfig(config.ApplicationName) - sessionConfig.ConnectionCreationTimeout = config.IOConnection.CreationTimeout + sessionConfig.ConnectionCreationTimeout = time.Duration(config.IOConnection.CreationTimeout) sessionConfig.ConnectionInitNumber = config.IOConnection.InitNumber - sessionConfig.ConnectionLifespan = config.IOConnection.Lifespan - sessionConfig.OperationTimeout = config.IOConnection.OperationTimeout - sessionConfig.ConnectionIdleTimeout = config.IOConnection.IdleTimeout + sessionConfig.ConnectionLifespan = time.Duration(config.IOConnection.Lifespan) + sessionConfig.OperationTimeout = time.Duration(config.IOConnection.OperationTimeout) + sessionConfig.ConnectionIdleTimeout = time.Duration(config.IOConnection.IdleTimeout) sessionConfig.ConnectionMaxNumber = config.IOConnection.MaxNumber sessionConfig.ConnectionMaxIdleNumber = config.IOConnection.MaxIdleNumber sessionConfig.TCPBufferSize = config.IOConnection.TCPBufferSize diff --git a/fs/fs.go b/fs/fs.go index 9727436..2010804 100644 --- a/fs/fs.go +++ b/fs/fs.go @@ -308,7 +308,7 @@ func (fs *FileSystem) RemoveFile(path string, force bool) error { defer fs.fileHandleMap.RemoveCloseEventHandler(eventHandlerID) - if util.WaitTimeout(&wg, fs.config.MetadataConnection.OperationTimeout) { + if util.WaitTimeout(&wg, time.Duration(fs.config.MetadataConnection.OperationTimeout)) { // timed out return xerrors.Errorf("failed to remove file, there are files still opened") } diff --git a/irods/types/duration.go b/irods/types/duration.go new file mode 100644 index 0000000..ab9639a --- /dev/null +++ b/irods/types/duration.go @@ -0,0 +1,67 @@ +package types + +import ( + "encoding/json" + "fmt" + "time" + + "golang.org/x/xerrors" +) + +// Duration is a replacement of time.Duration that supports JSON +type Duration time.Duration + +// MarshalJSON ... +func (d *Duration) MarshalJSON() ([]byte, error) { + return json.Marshal(time.Duration(*d).String()) +} + +// UnmarshalJSON ... +func (d *Duration) UnmarshalJSON(b []byte) error { + var v interface{} + if err := json.Unmarshal(b, &v); err != nil { + return xerrors.Errorf("failed to parse '%s' to time.Duration: %w", string(b), err) + } + switch value := v.(type) { + case float64: + *d = Duration(time.Duration(value)) + return nil + case string: + tmp, err := time.ParseDuration(value) + if err != nil { + return xerrors.Errorf("failed to parse '%s' to time.Duration: %w", string(b), err) + } + *d = Duration(tmp) + return nil + default: + return xerrors.Errorf("failed to parse '%s' to time.Duration", string(b)) + } +} + +// bug in YAMLv2, fixed in YAMLv3 +// // MarshalYAML ... +// func (d *Duration) MarshalYAML() (interface{}, error) { +// return time.Duration(*d).String(), nil +// } + +// UnmarshalYAML ... +func (d *Duration) UnmarshalYAML(unmarshal func(interface{}) error) error { + var tm string + if err := unmarshal(&tm); err != nil { + return xerrors.Errorf("failed to parse '%s' to time.Duration: %w", tm, err) + } + + lastChar := byte(tm[len(tm)-1]) + if lastChar >= '0' && lastChar <= '9' { + // ends with number, no units + tm = tm + "ns" + } + + td, err := time.ParseDuration(tm) + if err != nil { + return fmt.Errorf("failed to parse '%s' to time.Duration: %w", tm, err) + } + + *d = Duration(td) + return nil +} diff --git a/test/testcases/duration_test.go b/test/testcases/duration_test.go new file mode 100644 index 0000000..7c80952 --- /dev/null +++ b/test/testcases/duration_test.go @@ -0,0 +1,61 @@ +package testcases + +import ( + "testing" + "time" + + "github.com/cyverse/go-irodsclient/irods/types" + "github.com/stretchr/testify/assert" + "gopkg.in/yaml.v2" +) + +func TestDuration(t *testing.T) { + t.Run("test YAMLMarshal", testYAMLMarshal) + t.Run("test YAMLUnmarshalFromStringWithoutUnits", testYAMLUnmarshalFromStringWithoutUnits) + t.Run("test YAMLUnmarshalFromString", testYAMLUnmarshalFromString) +} + +func testYAMLMarshal(t *testing.T) { + d1 := types.Duration(5 * time.Minute) + + yamlBytes, err := yaml.Marshal(d1) + assert.NoError(t, err) + assert.NotEmpty(t, string(yamlBytes)) + + var d2 types.Duration + err = yaml.Unmarshal(yamlBytes, &d2) + assert.NoError(t, err) + assert.Equal(t, d1, d2) +} + +func testYAMLUnmarshalFromStringWithoutUnits(t *testing.T) { + v1 := []byte("60000000000") + + var d1 types.Duration + err := yaml.Unmarshal(v1, &d1) + assert.NoError(t, err) + assert.Equal(t, 1*time.Minute, time.Duration(d1)) + + v2 := []byte("6h60000000000") + + var d2 types.Duration + err = yaml.Unmarshal(v2, &d2) + assert.NoError(t, err) + assert.Equal(t, 6*time.Hour+1*time.Minute, time.Duration(d2)) +} + +func testYAMLUnmarshalFromString(t *testing.T) { + v1 := []byte("6m") + + var d1 types.Duration + err := yaml.Unmarshal(v1, &d1) + assert.NoError(t, err) + assert.Equal(t, 6*time.Minute, time.Duration(d1)) + + v2 := []byte("6h6m") + + var d2 types.Duration + err = yaml.Unmarshal(v2, &d2) + assert.NoError(t, err) + assert.Equal(t, 6*time.Hour+6*time.Minute, time.Duration(d2)) +}