Merge pull request #1442 from k8s-infra-cherrypick-robot/cherry-pick-1440-to-release-1.24

[release-1.24] fix: add VolumeStats cache to avoid massive statfs calls

Author: andyzhangx

pkg/azurefile/azurefile.go

@@ -199,6 +199,7 @@ type DriverOptions struct {
 	KubeAPIBurst                           int
 	AppendNoShareSockOption                bool
 	SkipMatchingTagCacheExpireInMinutes    int
+	VolStatsCacheExpireInMinutes           int
 }
 
 // Driver implements all interfaces of CSI drivers
@@ -243,6 +244,8 @@ type Driver struct {
 	skipMatchingTagCache *azcache.TimedCache
 	// a timed cache when resize file share failed due to account limit exceeded
 	resizeFileShareFailureCache *azcache.TimedCache
+	// a timed cache storing volume stats <volumeID, volumeStats>
+	volStatsCache *azcache.TimedCache
 }
 
 // NewDriver Creates a NewCSIDriver object. Assumes vendor version is equal to driver version &
@@ -299,6 +302,13 @@ func NewDriver(options *DriverOptions) *Driver {
 		klog.Fatalf("%v", err)
 	}
 
+	if options.VolStatsCacheExpireInMinutes <= 0 {
+		options.VolStatsCacheExpireInMinutes = 10 // default expire in 10 minutes
+	}
+	if driver.volStatsCache, err = azcache.NewTimedcache(time.Duration(options.VolStatsCacheExpireInMinutes)*time.Minute, getter); err != nil {
+		klog.Fatalf("%v", err)
+	}
+
 	return &driver
 }
 

pkg/azurefile/nodeserver.go

@@ -37,6 +37,8 @@ import (
 	"google.golang.org/grpc/status"
 
 	"golang.org/x/net/context"
+
+	azcache "sigs.k8s.io/cloud-provider-azure/pkg/cache"
 )
 
 // NodePublishVolume mount the volume from staging to target path
@@ -422,13 +424,48 @@ func (d *Driver) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeS
 		return nil, status.Error(codes.InvalidArgument, "NodeGetVolumeStats volume path was empty")
 	}
 
+	// check if the volume stats is cached
+	cache, err := d.volStatsCache.Get(req.VolumeId, azcache.CacheReadTypeDefault)
+	if err != nil {
+		return nil, status.Errorf(codes.Internal, err.Error())
+	}
+	if cache != nil {
+		resp := cache.(csi.NodeGetVolumeStatsResponse)
+		klog.V(6).Infof("NodeGetVolumeStats: volume stats for volume %s path %s is cached", req.VolumeId, req.VolumePath)
+		return &resp, nil
+	}
+
+	// fileShareName in volumeID may contain subPath, e.g. csi-shared-config/ASCP01/certs
+	// get the file share name without subPath from volumeID and check the cache again using new volumeID
+	var newVolID string
+	if _, accountName, fileShareName, _, secretNamespace, _, err := GetFileShareInfo(req.VolumeId); err == nil {
+		if splitStr := strings.Split(fileShareName, "/"); len(splitStr) > 1 {
+			fileShareName = splitStr[0]
+		}
+		// get new volumeID
+		if accountName != "" && fileShareName != "" {
+			newVolID = fmt.Sprintf(volumeIDTemplate, "", accountName, fileShareName, "", "", secretNamespace)
+		}
+	}
+
+	if cache, err = d.volStatsCache.Get(newVolID, azcache.CacheReadTypeDefault); err != nil {
+		return nil, status.Errorf(codes.Internal, err.Error())
+	}
+	if cache != nil {
+		resp := cache.(csi.NodeGetVolumeStatsResponse)
+		klog.V(6).Infof("NodeGetVolumeStats: volume stats for volume %s path %s is cached", req.VolumeId, req.VolumePath)
+		return &resp, nil
+	}
+
 	if _, err := os.Lstat(req.VolumePath); err != nil {
 		if os.IsNotExist(err) {
 			return nil, status.Errorf(codes.NotFound, "path %s does not exist", req.VolumePath)
 		}
 		return nil, status.Errorf(codes.Internal, "failed to stat file %s: %v", req.VolumePath, err)
 	}
 
+	klog.V(6).Infof("NodeGetVolumeStats: begin to get VolumeStats on volume %s path %s", req.VolumeId, req.VolumePath)
+
 	volumeMetrics, err := volume.NewMetricsStatFS(req.VolumePath).GetMetrics()
 	if err != nil {
 		return nil, status.Errorf(codes.Internal, "failed to get metrics: %v", err)
@@ -460,7 +497,7 @@ func (d *Driver) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeS
 		return nil, status.Errorf(codes.Internal, "failed to transform disk inodes used(%v)", volumeMetrics.InodesUsed)
 	}
 
-	return &csi.NodeGetVolumeStatsResponse{
+	resp := &csi.NodeGetVolumeStatsResponse{
 		Usage: []*csi.VolumeUsage{
 			{
 				Unit:      csi.VolumeUsage_BYTES,
@@ -475,7 +512,16 @@ func (d *Driver) NodeGetVolumeStats(ctx context.Context, req *csi.NodeGetVolumeS
 				Used:      inodesUsed,
 			},
 		},
-	}, nil
+	}
+
+	klog.V(6).Infof("NodeGetVolumeStats: volume stats for volume %s path %s is %v", req.VolumeId, req.VolumePath, resp)
+	// cache the volume stats per volume
+	d.volStatsCache.Set(req.VolumeId, *resp)
+	if newVolID != "" {
+		d.volStatsCache.Set(newVolID, *resp)
+	}
+
+	return resp, nil
 }
 
 // NodeExpandVolume node expand volume

pkg/azurefileplugin/main.go

@@ -56,6 +56,7 @@ var (
 	kubeAPIBurst                           = flag.Int("kube-api-burst", 50, "Burst to use while communicating with the kubernetes apiserver.")
 	appendNoShareSockOption                = flag.Bool("append-nosharesock-option", true, "Whether appending nosharesock option to smb mount command")
 	skipMatchingTagCacheExpireInMinutes    = flag.Int("skip-matching-tag-cache-expire-in-minutes", 30, "The cache expire time in minutes for skipMatchingTagCache")
+	volStatsCacheExpireInMinutes           = flag.Int("vol-stats-cache-expire-in-minutes", 10, "The cache expire time in minutes for volume stats cache")
 )
 
 func main() {
@@ -97,6 +98,7 @@ func handle() {
 		KubeAPIBurst:                           *kubeAPIBurst,
 		AppendNoShareSockOption:                *appendNoShareSockOption,
 		SkipMatchingTagCacheExpireInMinutes:    *skipMatchingTagCacheExpireInMinutes,
+		VolStatsCacheExpireInMinutes:           *volStatsCacheExpireInMinutes,
 	}
 	driver := azurefile.NewDriver(&driverOptions)
 	if driver == nil {