From 3d72e179e8b048a8d9e0264eb3fc7a169d82a602 Mon Sep 17 00:00:00 2001 From: Evgenii Baidakov Date: Tue, 21 Jan 2025 15:07:58 +0400 Subject: [PATCH] *: Clean tree service client Signed-off-by: Evgenii Baidakov --- api/layer/compound.go | 2 +- api/layer/tree_mock.go | 89 ----------------- api/layer/tree_service.go | 11 --- internal/neofs/tree.go | 198 -------------------------------------- 4 files changed, 1 insertion(+), 299 deletions(-) diff --git a/api/layer/compound.go b/api/layer/compound.go index f714d09d..4ed9b643 100644 --- a/api/layer/compound.go +++ b/api/layer/compound.go @@ -26,7 +26,7 @@ func (n *layer) GetObjectTaggingAndLock(ctx context.Context, objVersion *ObjectV } } - tags, _, err = n.treeService.GetObjectTaggingAndLock(ctx, objVersion.BktInfo, nodeVersion) + tags, err = n.treeService.GetObjectTagging(ctx, objVersion.BktInfo, nodeVersion) if err != nil { if errorsStd.Is(err, ErrNodeNotFound) { return nil, nil, s3errors.GetAPIError(s3errors.ErrNoSuchKey) diff --git a/api/layer/tree_mock.go b/api/layer/tree_mock.go index 26c10019..f5eb3ec5 100644 --- a/api/layer/tree_mock.go +++ b/api/layer/tree_mock.go @@ -5,7 +5,6 @@ import ( "context" "fmt" "slices" - "strings" "github.com/nspcc-dev/neofs-s3-gw/api/data" oid "github.com/nspcc-dev/neofs-sdk-go/object/id" @@ -21,12 +20,6 @@ type TreeServiceMock struct { parts map[string]map[int]*data.PartInfo } -func (t *TreeServiceMock) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) { - // TODO implement object tagging - lock, err := t.GetLock(ctx, bktInfo, objVersion.ID) - return nil, lock, err -} - func (t *TreeServiceMock) GetObjectTagging(_ context.Context, bktInfo *data.BucketInfo, nodeVersion *data.NodeVersion) (map[string]string, error) { cnrTagsMap, ok := t.tags[bktInfo.CID.EncodeToString()] if !ok { @@ -157,31 +150,6 @@ func (t *TreeServiceMock) GetLatestVersion(_ context.Context, bktInfo *data.Buck return nil, ErrNodeNotFound } -func (t *TreeServiceMock) GetLatestVersionsByPrefix(_ context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { - cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] - if !ok { - return nil, ErrNodeNotFound - } - - var result []*data.NodeVersion - - for key, versions := range cnrVersionsMap { - if !strings.HasPrefix(key, prefix) { - continue - } - - slices.SortFunc(versions, func(a, b *data.NodeVersion) int { - return cmp.Compare(a.ID, b.ID) - }) - - if len(versions) != 0 { - result = append(result, versions[len(versions)-1]) - } - } - - return result, nil -} - func (t *TreeServiceMock) GetUnversioned(_ context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) { cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] if !ok { @@ -242,40 +210,6 @@ func (t *TreeServiceMock) AddVersion(_ context.Context, bktInfo *data.BucketInfo return newVersion.ID, nil } -func (t *TreeServiceMock) RemoveVersion(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) error { - cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] - if !ok { - return ErrNodeNotFound - } - - for key, versions := range cnrVersionsMap { - for i, node := range versions { - if node.ID == nodeID { - cnrVersionsMap[key] = slices.Delete(versions, i, i+1) - return nil - } - } - } - - return ErrNodeNotFound -} - -func (t *TreeServiceMock) GetAllVersionsByPrefix(_ context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { - cnrVersionsMap, ok := t.versions[bktInfo.CID.EncodeToString()] - if !ok { - return nil, nil - } - - var result []*data.NodeVersion - for objName, versions := range cnrVersionsMap { - if strings.HasPrefix(objName, prefix) { - result = append(result, versions...) - } - } - - return result, nil -} - func (t *TreeServiceMock) CreateMultipartUpload(_ context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) { cnrMultipartsMap, ok := t.multiparts[bktInfo.CID.EncodeToString()] if !ok { @@ -453,26 +387,3 @@ LOOP: delete(t.parts, uploadID) return nil } - -func (t *TreeServiceMock) PutLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error { - cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()] - if !ok { - t.locks[bktInfo.CID.EncodeToString()] = map[uint64]*data.LockInfo{ - nodeID: lock, - } - return nil - } - - cnrLockMap[nodeID] = lock - - return nil -} - -func (t *TreeServiceMock) GetLock(_ context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) { - cnrLockMap, ok := t.locks[bktInfo.CID.EncodeToString()] - if !ok { - return nil, nil - } - - return cnrLockMap[nodeID], nil -} diff --git a/api/layer/tree_service.go b/api/layer/tree_service.go index 601ebcc3..ad4861f0 100644 --- a/api/layer/tree_service.go +++ b/api/layer/tree_service.go @@ -54,14 +54,8 @@ type TreeService interface { GetVersions(ctx context.Context, bktInfo *data.BucketInfo, objectName string) ([]*data.NodeVersion, error) GetLatestVersion(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) - GetLatestVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) - GetAllVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, objectName string) (*data.NodeVersion, error) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, newVersion *data.NodeVersion) (uint64, error) - RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) error - - PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error - GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) DeleteMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64) error @@ -79,11 +73,6 @@ type TreeService interface { // - [ErrPartListIsEmpty] if there is no parts in the upload id. GetPartByNumber(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, number int) (*data.PartInfo, error) GetPartsAfter(ctx context.Context, bktInfo *data.BucketInfo, multipartNodeID uint64, partID int) ([]*data.PartInfo, error) - - // Compound methods for optimizations - - // GetObjectTaggingAndLock unifies GetObjectTagging and GetLock methods in single tree service invocation. - GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) } var ( diff --git a/internal/neofs/tree.go b/internal/neofs/tree.go index f8aafa9c..d7c8d192 100644 --- a/internal/neofs/tree.go +++ b/internal/neofs/tree.go @@ -64,13 +64,6 @@ const ( homoHashKV = "HomoHash" elementsKV = "Elements" - // keys for lock. - isLockKV = "IsLock" - legalHoldOIDKV = "LegalHoldOID" - retentionOIDKV = "RetentionOID" - untilDateKV = "UntilDate" - isComplianceKV = "IsCompliance" - // keys for delete marker nodes. isDeleteMarkerKV = "IsDeleteMarker" ownerKV = "Owner" @@ -600,10 +593,6 @@ func pathFromName(objectName string) []string { return strings.Split(objectName, separator) } -func (c *TreeClient) GetLatestVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { - return c.getVersionsByPrefix(ctx, bktInfo, prefix, true) -} - func (c *TreeClient) determinePrefixNode(ctx context.Context, bktInfo *data.BucketInfo, treeID, prefix string) (uint64, string, error) { var rootID uint64 path := strings.Split(prefix, separator) @@ -722,117 +711,6 @@ func isIntermediate(node NodeResponse) bool { return node.GetMeta()[0].GetKey() == fileNameKV } -func (c *TreeClient) getSubTreeVersions(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, parentFilePath string, latestOnly bool) ([]*data.NodeVersion, error) { - subTree, err := c.getSubTree(ctx, bktInfo, versionTree, nodeID, maxGetSubTreeDepth) - if err != nil { - return nil, err - } - - var parentPrefix string - if parentFilePath != "" { // The root of subTree can also have a parent - parentPrefix = strings.TrimSuffix(parentFilePath, separator) + separator // To avoid 'foo//bar' - } - - var emptyOID oid.ID - var filepath string - namesMap := make(map[uint64]string, len(subTree)) - versions := make(map[string][]*data.NodeVersion, len(subTree)) - - for i, node := range subTree { - treeNode, fileName, err := parseTreeNode(node) - if err != nil { - continue - } - - if i != 0 { - if filepath, err = formFilePath(node, fileName, namesMap); err != nil { - return nil, fmt.Errorf("invalid node order: %w", err) - } - } else { - filepath = parentPrefix + fileName - namesMap[treeNode.ID] = filepath - } - - if treeNode.ObjID.Equals(emptyOID) { // The node can be intermediate but we still want to update namesMap - continue - } - - key := formLatestNodeKey(node.GetParentId(), fileName) - versionNodes, ok := versions[key] - if !ok { - versionNodes = []*data.NodeVersion{newNodeVersionFromTreeNode(filepath, treeNode)} - } else if !latestOnly { - versionNodes = append(versionNodes, newNodeVersionFromTreeNode(filepath, treeNode)) - } else if versionNodes[0].Timestamp <= treeNode.TimeStamp { - versionNodes[0] = newNodeVersionFromTreeNode(filepath, treeNode) - } - - versions[key] = versionNodes - } - - result := make([]*data.NodeVersion, 0, len(versions)) // consider use len(subTree) - for _, version := range versions { - if latestOnly && version[0].IsDeleteMarker() { - continue - } - result = append(result, version...) - } - - return result, nil -} - -func formFilePath(node *tree.GetSubTreeResponse_Body, fileName string, namesMap map[uint64]string) (string, error) { - parentPath, ok := namesMap[node.GetParentId()] - if !ok { - return "", fmt.Errorf("couldn't get parent path") - } - - filepath := parentPath + separator + fileName - namesMap[node.GetNodeId()] = filepath - - return filepath, nil -} - -func parseTreeNode(node *tree.GetSubTreeResponse_Body) (*TreeNode, string, error) { - treeNode, err := newTreeNode(node) - if err != nil { // invalid OID attribute - return nil, "", err - } - - fileName, ok := treeNode.FileName() - if !ok { - return nil, "", fmt.Errorf("doesn't contain FileName") - } - - return treeNode, fileName, nil -} - -func formLatestNodeKey(parentID uint64, fileName string) string { - return strconv.FormatUint(parentID, 10) + "." + fileName -} - -func (c *TreeClient) GetAllVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string) ([]*data.NodeVersion, error) { - return c.getVersionsByPrefix(ctx, bktInfo, prefix, false) -} - -func (c *TreeClient) getVersionsByPrefix(ctx context.Context, bktInfo *data.BucketInfo, prefix string, latestOnly bool) ([]*data.NodeVersion, error) { - prefixNodes, headPrefix, err := c.getSubTreeByPrefix(ctx, bktInfo, versionTree, prefix, latestOnly) - if err != nil { - return nil, err - } - - var result []*data.NodeVersion - for _, node := range prefixNodes { - versions, err := c.getSubTreeVersions(ctx, bktInfo, node.GetNodeId(), headPrefix, latestOnly) - if err != nil { - return nil, err - } - result = append(result, versions...) - } - - return result, nil -} - func (c *TreeClient) GetUnversioned(ctx context.Context, bktInfo *data.BucketInfo, filepath string) (*data.NodeVersion, error) { return c.getUnversioned(ctx, bktInfo, versionTree, filepath) } @@ -858,10 +736,6 @@ func (c *TreeClient) AddVersion(ctx context.Context, bktInfo *data.BucketInfo, v return c.addVersion(ctx, bktInfo, versionTree, version) } -func (c *TreeClient) RemoveVersion(ctx context.Context, bktInfo *data.BucketInfo, id uint64) error { - return c.removeNode(ctx, bktInfo, versionTree, id) -} - func (c *TreeClient) CreateMultipartUpload(ctx context.Context, bktInfo *data.BucketInfo, info *data.MultipartInfo) (uint64, error) { path := pathFromName(info.Key) meta := metaFromMultipart(info, path[len(path)-1]) @@ -1087,78 +961,6 @@ func (c *TreeClient) DeleteMultipartUpload(ctx context.Context, bktInfo *data.Bu return c.removeNode(ctx, bktInfo, systemTree, multipartNodeID) } -func (c *TreeClient) PutLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64, lock *data.LockInfo) error { - meta := map[string]string{isLockKV: "true"} - - if lock.IsLegalHoldSet() { - meta[legalHoldOIDKV] = lock.LegalHold().EncodeToString() - } - if lock.IsRetentionSet() { - meta[retentionOIDKV] = lock.Retention().EncodeToString() - meta[untilDateKV] = lock.UntilDate() - if lock.IsCompliance() { - meta[isComplianceKV] = "true" - } - } - - if lock.ID() == 0 { - _, err := c.addNode(ctx, bktInfo, versionTree, nodeID, meta) - return err - } - - return c.moveNode(ctx, bktInfo, versionTree, lock.ID(), nodeID, meta) -} - -func (c *TreeClient) GetLock(ctx context.Context, bktInfo *data.BucketInfo, nodeID uint64) (*data.LockInfo, error) { - lockNode, err := c.getTreeNode(ctx, bktInfo, nodeID, isLockKV) - if err != nil { - return nil, err - } - - return getLock(lockNode) -} - -func getLock(lockNode *TreeNode) (*data.LockInfo, error) { - if lockNode == nil { - return &data.LockInfo{}, nil - } - lockInfo := data.NewLockInfo(lockNode.ID) - - if legalHold, ok := lockNode.Get(legalHoldOIDKV); ok { - var legalHoldOID oid.ID - if err := legalHoldOID.DecodeString(legalHold); err != nil { - return nil, fmt.Errorf("invalid legal hold object id: %w", err) - } - lockInfo.SetLegalHold(legalHoldOID) - } - - if retention, ok := lockNode.Get(retentionOIDKV); ok { - var retentionOID oid.ID - if err := retentionOID.DecodeString(retention); err != nil { - return nil, fmt.Errorf("invalid retention object id: %w", err) - } - _, isCompliance := lockNode.Get(isComplianceKV) - untilDate, _ := lockNode.Get(untilDateKV) - lockInfo.SetRetention(retentionOID, untilDate, isCompliance) - } - - return lockInfo, nil -} - -func (c *TreeClient) GetObjectTaggingAndLock(ctx context.Context, bktInfo *data.BucketInfo, objVersion *data.NodeVersion) (map[string]string, *data.LockInfo, error) { - nodes, err := c.getTreeNodes(ctx, bktInfo, objVersion.ID, isTagKV, isLockKV) - if err != nil { - return nil, nil, err - } - - lockInfo, err := getLock(nodes[isLockKV]) - if err != nil { - return nil, nil, err - } - - return getObjectTagging(nodes[isTagKV]), lockInfo, nil -} - func (c *TreeClient) Close() error { if c.conn != nil { return c.conn.Close()