bug fixes; add SET refresh_interval

src/s3pool/conf/conf.go

@@ -0,0 +1,16 @@
+/*
+ *  S3pool - S3 cache on local disk
+ *  Copyright (c) 2019 CK Tan
+ *  [email protected]
+ *
+ *  S3Pool can be used for free under the GNU General Public License
+ *  version 3, where anything released into public must be open source,
+ *  or under a commercial license. The commercial license does not
+ *  cover derived or ported versions created by third parties under
+ *  GPL. To inquire about commercial license, please send email to
+ *  [email protected].
+ */
+package conf
+
+var Verbose int
+var RefreshInterval = 2

src/s3pool/main.go

@@ -115,12 +115,12 @@ func serve(c *tcp_server.Client, request string) {
 		reply, err = op.Pull(cmdargs)
 	case "GLOB":
 		reply, err = op.Glob(cmdargs)
-		if err != nil {
+		if err == nil {
 			mon.NotifyBucketmon <- cmdargs[0]
 		}
 	case "GLOBX":
 		reply, err = op.Globx(cmdargs)
-		if err != nil {
+		if err == nil {
 			mon.NotifyBucketmon <- cmdargs[0]
 		}
 	case "REFRESH":

src/s3pool/mon/bucketmon.go

@@ -14,31 +14,40 @@ package mon
 
 import (
 	"log"
+	"s3pool/conf"
 	"s3pool/op"
 	"time"
 )
 
 var NotifyBucketmon chan<- string
 
 func Bucketmon() {
-	const REFRESHINTERVAL = 15 // minutes
 	ch := make(chan string, 10)
 	NotifyBucketmon = ch
 
 	go func() {
-		tick := time.Tick(REFRESHINTERVAL * time.Minute)
+		countdown := conf.RefreshInterval
+		tick := time.Tick(time.Minute)
 		bktmap := make(map[string](bool))
 		for {
 			select {
 			case bkt := <-ch:
+				log.Printf("Bucket notify %s\n", bkt)
 				bktmap[bkt] = true
 			case <-tick:
+				log.Printf("BUCKETMON %d", countdown)
+				countdown--
+				if countdown > 0 {
+					continue
+				}
+				countdown = conf.RefreshInterval
 				for bkt := range bktmap {
-					log.Println("bucketmon refresh", bkt)
+					log.Println("BUCKETMON refresh", bkt)
 					_, err := op.Refresh([]string{bkt})
 					if err != nil {
 						log.Printf("WARNING: autorefresh %s failed: %v\n", bkt, err)
 					}
+					log.Println("BUCKETMON fin", bkt)
 				}
 			}
 		}

src/s3pool/op/refresh.go

@@ -14,10 +14,8 @@ package op
 
 import (
 	"errors"
-	"log"
 	"s3pool/cat"
 	"s3pool/strlock"
-	"time"
 )
 
 /*
@@ -40,13 +38,15 @@ func Refresh(args []string) (string, error) {
 	defer strlock.Unlock(lockname)
 
 	numItems := 0
-	log.Println("REFRESH start on", bucket)
-	startTime := time.Now()
-	defer func() {
-		endTime := time.Now()
-		elapsed := int(endTime.Sub(startTime) / time.Millisecond)
-		log.Printf("REFRESH fin on %s, %d items, elapsed %d ms\n", bucket, numItems, elapsed)
-	}()
+	/*
+		log.Println("REFRESH start on", bucket)
+		startTime := time.Now()
+		defer func() {
+			endTime := time.Now()
+			elapsed := int(endTime.Sub(startTime) / time.Millisecond)
+			log.Printf("REFRESH fin on %s, %d items, elapsed %d ms\n", bucket, numItems, elapsed)
+		}()
+	*/
 
 	key := make([]string, 0, 100)
 	etag := make([]string, 0, 100)

src/s3pool/op/s3.go

@@ -22,6 +22,7 @@ import (
 	"os"
 	"os/exec"
 	"s3pool/cat"
+	"s3pool/conf"
 	"s3pool/strlock"
 	"strings"
 )
@@ -36,7 +37,7 @@ type ListCollection struct {
 }
 
 func s3ListObjects(bucket string, notify func(key, etag string)) error {
-	if Verbose > 0 {
+	if conf.Verbose > 0 {
 		log.Println("s3 list-objects", bucket)
 	}
 
@@ -131,7 +132,7 @@ func extractETag(path string) string {
 //   aws s3api get-object --bucket BUCKET --key KEY --if-none-match ETAG tmppath
 //
 func s3GetObject(bucket string, key string, force bool) (string, error) {
-	if Verbose > 0 {
+	if conf.Verbose > 0 {
 		log.Println("s3 get-objects", bucket, key)
 	}
 
@@ -155,13 +156,13 @@ func s3GetObject(bucket string, key string, force bool) (string, error) {
 
 	// If etag did not change, don't go fetch it
 	if etag != "" && etag == catetag && !force {
-		if Verbose > 0 {
+		if conf.Verbose > 0 {
 			log.Println(" ... cache hit:", key)
 		}
 		return path, nil
 	}
 
-	if Verbose > 0 {
+	if conf.Verbose > 0 {
 		log.Println(" ... cache miss:", key)
 		if catetag == "" {
 			log.Println(" ... missing catalog entry")
@@ -190,7 +191,7 @@ func s3GetObject(bucket string, key string, force bool) (string, error) {
 	notModified := strings.Contains(errstr, "Not Modified") && strings.Contains(errstr, "(304)")
 	if notModified {
 		// File was cached and was not modified at source
-		if Verbose > 0 {
+		if conf.Verbose > 0 {
 			log.Println(" ... file not modified")
 		}
 		//log.Println("   ... etag", etag)
@@ -232,7 +233,7 @@ func s3GetObject(bucket string, key string, force bool) (string, error) {
 // aws s3api put-object
 //
 func s3PutObject(bucket, key, fname string) error {
-	if Verbose > 0 {
+	if conf.Verbose > 0 {
 		log.Println("s3 put-object", bucket, key, fname)
 	}
 

src/s3pool/op/set.go

@@ -0,0 +1,51 @@
+/*
+ *  S3pool - S3 cache on local disk
+ *  Copyright (c) 2019 CK Tan
+ *  [email protected]
+ *
+ *  S3Pool can be used for free under the GNU General Public License
+ *  version 3, where anything released into public must be open source,
+ *  or under a commercial license. The commercial license does not
+ *  cover derived or ported versions created by third parties under
+ *  GPL. To inquire about commercial license, please send email to
+ *  [email protected].
+ */
+package op
+
+import (
+	"errors"
+	"s3pool/conf"
+	"strconv"
+	"strings"
+)
+
+func strtobool(s string) bool {
+	return s == "true" || s == "1" || s == "on"
+}
+
+func Set(args []string) (string, error) {
+	if len(args) != 2 {
+		return "", errors.New("expects 2 arguments for SET")
+	}
+	varname, varvalue := strings.ToLower(args[0]), strings.ToLower(args[1])
+
+	if varname == "verbose" {
+		i, err := strconv.Atoi(varvalue)
+		if err != nil {
+			return "", err
+		}
+		conf.Verbose = i
+		return "\n", nil
+	}
+
+	if varname == "refresh_interval" {
+		i, err := strconv.Atoi(varvalue)
+		if err != nil {
+			return "", err
+		}
+		conf.RefreshInterval = i
+		return "\n", nil
+	}
+
+	return "", errors.New("Unknown var name")
+}