Streamline getobject (#1085)

nspcc-dev · Feb 27, 2025 · 91bb004 · 91bb004
2 parents 30f920c + 49be936
commit 91bb004
Show file tree

Hide file tree

Showing 13 changed files with 491 additions and 159 deletions.
diff --git a/api/data/info.go b/api/data/info.go
@@ -63,6 +63,7 @@ type (
 		Owner          user.ID
 		OwnerPublicKey keys.PublicKey
 		Headers        map[string]string
+		Version        string
 	}
 
 	// ObjectListResponseContent holds response data for object listing.

diff --git a/api/data/tree.go b/api/data/tree.go
@@ -41,6 +41,13 @@ type ExtendedObjectInfo struct {
 	IsLatest    bool
 }
 
+// ComprehensiveObjectInfo represents metasearch result for object, with tags and lock data.
+type ComprehensiveObjectInfo struct {
+	ID       oid.ID
+	TagSet   map[string]string
+	LockInfo *LockInfo
+}
+
 func (e ExtendedObjectInfo) Version() string {
 	if e.NodeVersion.IsUnversioned {
 		return UnversionedObjectVersionID

diff --git a/api/handler/get.go b/api/handler/get.go
@@ -2,6 +2,7 @@ package handler
 
 import (
 	"fmt"
+	"io"
 	"net/http"
 	"net/url"
 	"strconv"
@@ -22,6 +23,10 @@ type conditionalArgs struct {
 	IfNoneMatch       string
 }
 
+const (
+	defaultBufferSize = 128 * 1024
+)
+
 func fetchRangeHeader(headers http.Header, fullSize uint64) (*layer.RangeParams, error) {
 	const prefix = "bytes="
 	rangeHeader := headers.Get("Range")
@@ -77,8 +82,7 @@ func addSSECHeaders(responseHeader http.Header, requestHeader http.Header) {
 	responseHeader.Set(api.AmzServerSideEncryptionCustomerKeyMD5, requestHeader.Get(api.AmzServerSideEncryptionCustomerKeyMD5))
 }
 
-func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.ExtendedObjectInfo, tagSetLength int, isBucketUnversioned bool) {
-	info := extendedInfo.ObjectInfo
+func writeHeaders(h http.Header, requestHeader http.Header, info *data.ObjectInfo, tagSetLength int, isBucketUnversioned bool) {
 	if len(info.ContentType) > 0 && h.Get(api.ContentType) == "" {
 		h.Set(api.ContentType, info.ContentType)
 	}
@@ -95,7 +99,7 @@ func writeHeaders(h http.Header, requestHeader http.Header, extendedInfo *data.E
 	h.Set(api.AmzTaggingCount, strconv.Itoa(tagSetLength))
 
 	if !isBucketUnversioned {
-		h.Set(api.AmzVersionID, extendedInfo.Version())
+		h.Set(api.AmzVersionID, info.Version)
 	}
 
 	if cacheControl := info.Headers[api.CacheControl]; cacheControl != "" {
@@ -138,12 +142,24 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 		VersionID: reqInfo.URL.Query().Get(api.QueryVersionID),
 	}
 
-	extendedInfo, err := h.obj.GetExtendedObjectInfo(r.Context(), p)
+	comprehensiveObjectInfo, err := h.obj.ComprehensiveObjectInfo(r.Context(), p)
 	if err != nil {
 		h.logAndSendError(w, "could not find object", reqInfo, err)
 		return
 	}
-	info := extendedInfo.ObjectInfo
+
+	objectWithPayloadReader, err := h.obj.GetObjectWithPayloadReader(r.Context(), &layer.GetObjectWithPayloadReaderParams{
+		Owner:   bktInfo.Owner,
+		BktInfo: bktInfo,
+		Object:  comprehensiveObjectInfo.ID,
+	})
+
+	if err != nil {
+		h.logAndSendError(w, "could not get object meta", reqInfo, err)
+		return
+	}
+
+	info := objectWithPayloadReader.ObjectInfo
 
 	if err = checkPreconditions(info, conditional); err != nil {
 		h.logAndSendError(w, "precondition failed", reqInfo, err)
@@ -180,46 +196,52 @@ func (h *handler) GetObjectHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	t := &layer.ObjectVersion{
-		BktInfo:    bktInfo,
-		ObjectName: info.Name,
-	}
-
-	if bktSettings.VersioningEnabled() {
-		t.VersionID = info.VersionID()
-	}
-
-	tagSet, lockInfo, err := h.obj.GetObjectTaggingAndLock(r.Context(), t, extendedInfo.NodeVersion)
-	if err != nil && !s3errors.IsS3Error(err, s3errors.ErrNoSuchKey) {
-		h.logAndSendError(w, "could not get object meta data", reqInfo, err)
-		return
-	}
-
 	if layer.IsAuthenticatedRequest(r.Context()) {
 		overrideResponseHeaders(w.Header(), reqInfo.URL.Query())
 	}
 
-	if err = h.setLockingHeaders(bktInfo, lockInfo, w.Header()); err != nil {
+	if err = h.setLockingHeaders(bktInfo, comprehensiveObjectInfo.LockInfo, w.Header()); err != nil {
 		h.logAndSendError(w, "could not get locking info", reqInfo, err)
 		return
 	}
 
-	writeHeaders(w.Header(), r.Header, extendedInfo, len(tagSet), bktSettings.Unversioned())
+	writeHeaders(w.Header(), r.Header, info, len(comprehensiveObjectInfo.TagSet), bktSettings.Unversioned())
 	if params != nil {
 		writeRangeHeaders(w, params, info.Size)
 	} else {
 		w.WriteHeader(http.StatusOK)
 	}
 
-	getParams := &layer.GetObjectParams{
-		ObjectInfo: info,
-		Writer:     w,
-		Range:      params,
-		BucketInfo: bktInfo,
-		Encryption: encryptionParams,
+	if params != nil || encryptionParams.Enabled() {
+		// unfortunately this reader is useless for us in this case, we have to re-read another one.
+		_ = objectWithPayloadReader.Payload.Close()
+
+		getParams := &layer.GetObjectParams{
+			ObjectInfo: info,
+			Writer:     w,
+			Range:      params,
+			BucketInfo: bktInfo,
+			Encryption: encryptionParams,
+		}
+		if err = h.obj.GetObject(r.Context(), getParams); err != nil {
+			h.logAndSendError(w, "could not get object", reqInfo, err)
+		}
+
+		return
+	}
+
+	var bufferSize = min(info.Size, defaultBufferSize)
+	if bufferSize == 0 {
+		bufferSize = defaultBufferSize
 	}
-	if err = h.obj.GetObject(r.Context(), getParams); err != nil {
-		h.logAndSendError(w, "could not get object", reqInfo, err)
+
+	buf := make([]byte, bufferSize)
+	if _, err = io.CopyBuffer(w, objectWithPayloadReader.Payload, buf); err != nil {
+		h.logAndSendError(w, "could write object output", reqInfo, err)
+	}
+
+	if err = objectWithPayloadReader.Payload.Close(); err != nil {
+		h.logAndSendError(w, "close output", reqInfo, err)
 	}
 }
 

diff --git a/api/handler/head.go b/api/handler/head.go
@@ -85,7 +85,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 		t.VersionID = ""
 	}
 
-	tagSet, lockInfo, err := h.obj.GetObjectTaggingAndLock(r.Context(), t, extendedInfo.NodeVersion)
+	tagSet, lockInfo, err := h.obj.GetObjectTaggingAndLock(r.Context(), t)
 	if err != nil && !s3errors.IsS3Error(err, s3errors.ErrNoSuchKey) {
 		h.logAndSendError(w, "could not get object meta data", reqInfo, err)
 		return
@@ -113,7 +113,7 @@ func (h *handler) HeadObjectHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	writeHeaders(w.Header(), r.Header, extendedInfo, len(tagSet), bktSettings.Unversioned())
+	writeHeaders(w.Header(), r.Header, extendedInfo.ObjectInfo, len(tagSet), bktSettings.Unversioned())
 	w.WriteHeader(http.StatusOK)
 }
 

diff --git a/api/layer/compound.go b/api/layer/compound.go
@@ -8,7 +8,7 @@ import (
 	"github.com/nspcc-dev/neofs-s3-gw/api/s3errors"
 )
 
-func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion, nodeVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) {
+func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectVersion) (map[string]string, *data.LockInfo, error) {
 	var err error
 	owner := n.Owner(ctx)