From d820cc30fa768dd990f888d2a3779e07b2ec2e25 Mon Sep 17 00:00:00 2001 From: CK Tan Date: Wed, 13 Nov 2019 18:24:52 -0800 Subject: [PATCH] bug fixes; add SET refresh_interval --- src/s3pool/conf/conf.go | 16 ++++++++++++ src/s3pool/main.go | 4 +-- src/s3pool/mon/bucketmon.go | 15 ++++++++--- src/s3pool/op/refresh.go | 18 ++++++------- src/s3pool/op/s3.go | 13 +++++----- src/s3pool/op/set.go | 51 +++++++++++++++++++++++++++++++++++++ 6 files changed, 97 insertions(+), 20 deletions(-) create mode 100644 src/s3pool/conf/conf.go create mode 100644 src/s3pool/op/set.go diff --git a/src/s3pool/conf/conf.go b/src/s3pool/conf/conf.go new file mode 100644 index 0000000..2e4955d --- /dev/null +++ b/src/s3pool/conf/conf.go @@ -0,0 +1,16 @@ +/* + * S3pool - S3 cache on local disk + * Copyright (c) 2019 CK Tan + * cktanx@gmail.com + * + * S3Pool can be used for free under the GNU General Public License + * version 3, where anything released into public must be open source, + * or under a commercial license. The commercial license does not + * cover derived or ported versions created by third parties under + * GPL. To inquire about commercial license, please send email to + * cktanx@gmail.com. + */ +package conf + +var Verbose int +var RefreshInterval = 2 diff --git a/src/s3pool/main.go b/src/s3pool/main.go index 60df4ee..22286d5 100644 --- a/src/s3pool/main.go +++ b/src/s3pool/main.go @@ -115,12 +115,12 @@ func serve(c *tcp_server.Client, request string) { reply, err = op.Pull(cmdargs) case "GLOB": reply, err = op.Glob(cmdargs) - if err != nil { + if err == nil { mon.NotifyBucketmon <- cmdargs[0] } case "GLOBX": reply, err = op.Globx(cmdargs) - if err != nil { + if err == nil { mon.NotifyBucketmon <- cmdargs[0] } case "REFRESH": diff --git a/src/s3pool/mon/bucketmon.go b/src/s3pool/mon/bucketmon.go index 291f921..3aa4dc8 100644 --- a/src/s3pool/mon/bucketmon.go +++ b/src/s3pool/mon/bucketmon.go @@ -14,6 +14,7 @@ package mon import ( "log" + "s3pool/conf" "s3pool/op" "time" ) @@ -21,24 +22,32 @@ import ( var NotifyBucketmon chan<- string func Bucketmon() { - const REFRESHINTERVAL = 15 // minutes ch := make(chan string, 10) NotifyBucketmon = ch go func() { - tick := time.Tick(REFRESHINTERVAL * time.Minute) + countdown := conf.RefreshInterval + tick := time.Tick(time.Minute) bktmap := make(map[string](bool)) for { select { case bkt := <-ch: + log.Printf("Bucket notify %s\n", bkt) bktmap[bkt] = true case <-tick: + log.Printf("BUCKETMON %d", countdown) + countdown-- + if countdown > 0 { + continue + } + countdown = conf.RefreshInterval for bkt := range bktmap { - log.Println("bucketmon refresh", bkt) + log.Println("BUCKETMON refresh", bkt) _, err := op.Refresh([]string{bkt}) if err != nil { log.Printf("WARNING: autorefresh %s failed: %v\n", bkt, err) } + log.Println("BUCKETMON fin", bkt) } } } diff --git a/src/s3pool/op/refresh.go b/src/s3pool/op/refresh.go index 85c6e73..de59df6 100644 --- a/src/s3pool/op/refresh.go +++ b/src/s3pool/op/refresh.go @@ -14,10 +14,8 @@ package op import ( "errors" - "log" "s3pool/cat" "s3pool/strlock" - "time" ) /* @@ -40,13 +38,15 @@ func Refresh(args []string) (string, error) { defer strlock.Unlock(lockname) numItems := 0 - log.Println("REFRESH start on", bucket) - startTime := time.Now() - defer func() { - endTime := time.Now() - elapsed := int(endTime.Sub(startTime) / time.Millisecond) - log.Printf("REFRESH fin on %s, %d items, elapsed %d ms\n", bucket, numItems, elapsed) - }() + /* + log.Println("REFRESH start on", bucket) + startTime := time.Now() + defer func() { + endTime := time.Now() + elapsed := int(endTime.Sub(startTime) / time.Millisecond) + log.Printf("REFRESH fin on %s, %d items, elapsed %d ms\n", bucket, numItems, elapsed) + }() + */ key := make([]string, 0, 100) etag := make([]string, 0, 100) diff --git a/src/s3pool/op/s3.go b/src/s3pool/op/s3.go index a90d8ca..780170d 100644 --- a/src/s3pool/op/s3.go +++ b/src/s3pool/op/s3.go @@ -22,6 +22,7 @@ import ( "os" "os/exec" "s3pool/cat" + "s3pool/conf" "s3pool/strlock" "strings" ) @@ -36,7 +37,7 @@ type ListCollection struct { } func s3ListObjects(bucket string, notify func(key, etag string)) error { - if Verbose > 0 { + if conf.Verbose > 0 { log.Println("s3 list-objects", bucket) } @@ -131,7 +132,7 @@ func extractETag(path string) string { // aws s3api get-object --bucket BUCKET --key KEY --if-none-match ETAG tmppath // func s3GetObject(bucket string, key string, force bool) (string, error) { - if Verbose > 0 { + if conf.Verbose > 0 { log.Println("s3 get-objects", bucket, key) } @@ -155,13 +156,13 @@ func s3GetObject(bucket string, key string, force bool) (string, error) { // If etag did not change, don't go fetch it if etag != "" && etag == catetag && !force { - if Verbose > 0 { + if conf.Verbose > 0 { log.Println(" ... cache hit:", key) } return path, nil } - if Verbose > 0 { + if conf.Verbose > 0 { log.Println(" ... cache miss:", key) if catetag == "" { log.Println(" ... missing catalog entry") @@ -190,7 +191,7 @@ func s3GetObject(bucket string, key string, force bool) (string, error) { notModified := strings.Contains(errstr, "Not Modified") && strings.Contains(errstr, "(304)") if notModified { // File was cached and was not modified at source - if Verbose > 0 { + if conf.Verbose > 0 { log.Println(" ... file not modified") } //log.Println(" ... etag", etag) @@ -232,7 +233,7 @@ func s3GetObject(bucket string, key string, force bool) (string, error) { // aws s3api put-object // func s3PutObject(bucket, key, fname string) error { - if Verbose > 0 { + if conf.Verbose > 0 { log.Println("s3 put-object", bucket, key, fname) } diff --git a/src/s3pool/op/set.go b/src/s3pool/op/set.go new file mode 100644 index 0000000..c51112d --- /dev/null +++ b/src/s3pool/op/set.go @@ -0,0 +1,51 @@ +/* + * S3pool - S3 cache on local disk + * Copyright (c) 2019 CK Tan + * cktanx@gmail.com + * + * S3Pool can be used for free under the GNU General Public License + * version 3, where anything released into public must be open source, + * or under a commercial license. The commercial license does not + * cover derived or ported versions created by third parties under + * GPL. To inquire about commercial license, please send email to + * cktanx@gmail.com. + */ +package op + +import ( + "errors" + "s3pool/conf" + "strconv" + "strings" +) + +func strtobool(s string) bool { + return s == "true" || s == "1" || s == "on" +} + +func Set(args []string) (string, error) { + if len(args) != 2 { + return "", errors.New("expects 2 arguments for SET") + } + varname, varvalue := strings.ToLower(args[0]), strings.ToLower(args[1]) + + if varname == "verbose" { + i, err := strconv.Atoi(varvalue) + if err != nil { + return "", err + } + conf.Verbose = i + return "\n", nil + } + + if varname == "refresh_interval" { + i, err := strconv.Atoi(varvalue) + if err != nil { + return "", err + } + conf.RefreshInterval = i + return "\n", nil + } + + return "", errors.New("Unknown var name") +}