Use io.ReadAll() and add more persistence-related logs (#3140)

- Use io.ReadAll to read from object storage
- Add some more debug-level logs around persistent storage

<!-- end of auto-generated comment: release notes by coderabbit.ai -->

pkg/objectstorage/object-storage.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"io"
 	"strconv"
 	"sync"
 	"time"
@@ -95,9 +96,10 @@ func (o *ObjectStorage) Get(ctx context.Context, key string) (olricstorage.Entry
 	reader, err := obj.NewReader(timeoutCtx)
 	if err != nil {
 		if errors.Is(err, storage.ErrObjectNotExist) {
+			log.Debug().Err(err).Str("key", key).Msg("Requested object not found in the object storage")
 			return nil, ErrKeyNotFound
 		}
-		log.Error().Err(err).Msg("Failed to create object storage reader")
+		log.Error().Err(err).Str("key", key).Msg("Failed to create object storage reader")
 		return nil, err
 	}
 
@@ -108,13 +110,14 @@ func (o *ObjectStorage) Get(ctx context.Context, key string) (olricstorage.Entry
 		}
 	}()
 
-	data := make([]byte, reader.Attrs.Size)
-	_, err = reader.Read(data)
+	data, err := io.ReadAll(reader)
 	if err != nil {
 		log.Error().Err(err).Msg("Failed to read object storage object")
 		return nil, err
 	}
 
+	log.Debug().Str("key", key).Msg("Retrieved data from object storage")
+
 	entry := &PersistentEntry{key: key, value: data}
 	attrs, err := obj.Attrs(timeoutCtx)
 	if err != nil {

pkg/objectstorage/persistent-dmap.go

@@ -91,6 +91,8 @@ func (o *ObjectStorageBackedDMap) Get(ctx context.Context, key string) (*olric.G
 		// Some error from in-memory cache.
 		return nil, err
 	}
+
+	log.Debug().Str("key", key).Msg("Key not found in in-memory cache")
 	// Key not found in in-memory cache. Need to check backing storage.
 	metric, ready := o.getMissesTotalMetric(metrics.PersistentCacheTypeInMemory)
 	if ready {
@@ -122,6 +124,7 @@ func (o *ObjectStorageBackedDMap) Get(ctx context.Context, key string) (*olric.G
 		return nil, innerErr
 	}
 
+	log.Debug().Str("key", key).Msg("Entry from storage saved in in-memory cache and response returned")
 	return olric.NewResponse(entry), nil
 }