Implement a user:group ownership preserving flag

The `--preserve-ownership` flag will query the user and group and store that in S3 as metadata.

On Windows this stores the SID, on linux this stores the uid/gid.

** Note this has been rebased as of 2024/03/04

command/cp.go

@@ -117,9 +117,15 @@ Examples:
 	25. Upload a file to S3 preserving the timestamp on disk
 		 > s5cmd --preserve-timestamp myfile.css.br s3://bucket/
 
-	26. Download a file from S3 preserving the timestamp it was originally uplaoded with
+	26. Download a file from S3 preserving the timestamp it was originally uploaded with
 		 > s5cmd --preserve-timestamp s3://bucket/myfile.css.br myfile.css.br
 
+	27. Upload a file to S3 preserving the ownership of files
+		 > s5cmd --preserve-ownership myfile.css.br s3://bucket/
+
+	28. Download a file from S3 preserving the ownership it was originally uploaded with
+		 > s5cmd --preserve-ownership s3://bucket/myfile.css.br myfile.css.br
+
 `
 
 func NewSharedFlags() []cli.Flag {
@@ -218,6 +224,10 @@ func NewSharedFlags() []cli.Flag {
 			Name:  "preserve-timestamp",
 			Usage: "preserve the timestamp on disk while uploading and set the timestamp from s3 while downloading.",
 		},
+		&cli.BoolFlag{
+			Name:  "preserve-ownership",
+			Usage: "preserve the ownership (owner/group) on disk while uploading and set the ownership from s3 while downloading.",
+		},
 	}
 }
 
@@ -317,6 +327,7 @@ type Copy struct {
 	contentDisposition    string
 	metadata              map[string]string
 	preserveTimestamp     bool
+	preserveOwnership     bool
 	showProgress          bool
 	progressbar           progressbar.ProgressBar
 
@@ -397,6 +408,7 @@ func NewCopy(c *cli.Context, deleteSource bool) (*Copy, error) {
 		showProgress:          c.Bool("show-progress"),
 		progressbar:           commandProgressBar,
 		preserveTimestamp:     c.Bool("preserve-timestamp"),
+		preserveOwnership:     c.Bool("preserve-ownership"),
 
 		// region settings
 		srcRegion: c.String("source-region"),
@@ -661,6 +673,31 @@ func (c Copy) doDownload(ctx context.Context, srcurl *url.URL, dsturl *url.URL)
 		return err
 	}
 
+	if c.preserveOwnership {
+		obj, err := srcClient.Stat(ctx, srcurl)
+		if err != nil {
+			return err
+		}
+		// SetFileUserGroup may return an InvalidOwnershipFormatError which signifies that it cannot
+		//		understand the UserId or GroupId format.
+		// This is most common when a file is being ported across windows/linux.
+		// We aren't implementing a fix for it here, just a note that it cannot be resolved.
+		err = storage.SetFileUserGroup(dsturl.Absolute(), obj.UserId, obj.GroupId)
+		if err != nil {
+			invalidOwnershipFormat := &storage.InvalidOwnershipFormatError{}
+			if errors.As(err, &invalidOwnershipFormat) {
+				msg := log.ErrorMessage{
+					Operation: c.op,
+					Command:   c.fullCommand,
+					Err:       fmt.Sprintf("UserId: %s or GroupId: %s are not valid on this operating system.", obj.UserId, obj.GroupId),
+				}
+				log.Debug(msg)
+			}
+
+			return err
+		}
+	}
+
 	if c.preserveTimestamp {
 		obj, err := srcClient.Stat(ctx, srcurl)
 		if err != nil {
@@ -734,6 +771,14 @@ func (c Copy) doUpload(ctx context.Context, srcurl *url.URL, dsturl *url.URL, ex
 		storage.SetMetadataTimestamp(metadata, aTime, mTime, cTime)
 	}
 
+	if c.preserveOwnership {
+		userId, groupId, err := storage.GetFileUserGroup(srcurl.Absolute())
+		if err != nil {
+			return err
+		}
+		metadata.SetPreserveOwnership(userId, groupId)
+	}
+
 	if c.contentType != "" {
 		metadata.ContentType = c.contentType
 	} else {

command/sync.go

@@ -132,6 +132,7 @@ type Sync struct {
 	sizeOnly          bool
 	exitOnError       bool
 	preserveTimestamp bool
+	preserveOwnership bool
 
 	// s3 options
 	storageOpts storage.Options
@@ -157,6 +158,7 @@ func NewSync(c *cli.Context) Sync {
 		sizeOnly:          c.Bool("size-only"),
 		exitOnError:       c.Bool("exit-on-error"),
 		preserveTimestamp: c.Bool("preserve-timestamp"),
+		preserveOwnership: c.Bool("preserve-ownership"),
 
 		// flags
 		followSymlinks: !c.Bool("no-follow-symlinks"),
@@ -452,6 +454,9 @@ func (s Sync) planRun(
 	defaultFlags := map[string]interface{}{
 		"raw": true,
 	}
+	if s.preserveOwnership {
+		defaultFlags["preserve-ownership"] = s.preserveOwnership
+	}
 	if s.preserveTimestamp {
 		defaultFlags["preserve-timestamp"] = s.preserveTimestamp
 	}

go.mod

@@ -3,6 +3,7 @@ module github.com/peak/s5cmd/v2
 go 1.19
 
 require (
+	github.com/Microsoft/go-winio v0.6.1
 	github.com/aws/aws-sdk-go v1.44.256
 	github.com/cheggaaa/pb/v3 v3.1.4
 	github.com/golang/mock v1.6.0
@@ -15,6 +16,7 @@ require (
 	github.com/lanrat/extsort v1.0.0
 	github.com/termie/go-shutil v0.0.0-20140729215957-bcacb06fecae
 	github.com/urfave/cli/v2 v2.11.2
+	golang.org/x/sys v0.7.0
 	gotest.tools/v3 v3.0.3
 )
 
@@ -36,8 +38,8 @@ require (
 	github.com/shabbyrobe/gocovmerge v0.0.0-20190829150210-3e036491d500 // indirect
 	github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
+	golang.org/x/mod v0.10.0 // indirect
 	golang.org/x/sync v0.1.0 // indirect
-	golang.org/x/sys v0.7.0 // indirect
 	golang.org/x/tools v0.8.0 // indirect
 	gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 // indirect
 	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect

go.sum

@@ -1,3 +1,5 @@
+github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow=
+github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM=
 github.com/VividCortex/ewma v1.2.0 h1:f58SaIzcDXrSy3kWaHNvuJgJ3Nmz59Zji6XoJR/q1ow=
 github.com/VividCortex/ewma v1.2.0/go.mod h1:nz4BbCtbLyFDeC9SUHbtcT5644juEuWfUAUnGx7j5l4=
 github.com/aws/aws-sdk-go v1.44.256 h1:O8VH+bJqgLDguqkH/xQBFz5o/YheeZqgcOYIgsTVWY4=
@@ -83,6 +85,7 @@ golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5y
 golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4=
 golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
+golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk=
 golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=

storage/fs.go

@@ -12,6 +12,14 @@ import (
 	"github.com/peak/s5cmd/v2/storage/url"
 )
 
+type InvalidOwnershipFormatError struct {
+	Err error
+}
+
+func (e *InvalidOwnershipFormatError) Error() string {
+	return fmt.Sprintf("InvalidOwnershipFormatError: %v\n", e.Err)
+}
+
 // Filesystem is the Storage implementation of a local filesystem.
 type Filesystem struct {
 	dryRun bool

storage/fs_darwin.go

@@ -45,3 +45,41 @@ func SetFileTime(filename string, accessTime, modificationTime, creationTime tim
 	}
 	return nil
 }
+
+// GetFileUserGroup will take a filename and return the userId and groupId associated with it.
+//
+//	On windows this is in the format of a SID, on linux/darwin this is in the format of a UID/GID.
+func GetFileUserGroup(filename string) (username, group string, err error) {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return "", "", err
+	}
+
+	stat := info.Sys().(*syscall.Stat_t)
+
+	username := strconv.Itoa(stat.Uid)
+	group := strconv.Itoa(stat.Gid)
+	return username, group, nil
+}
+
+// SetFileUserGroup will set the UserId and GroupId on a filename.
+//
+//	If the UserId/GroupId format does not match the platform, it will return an InvalidOwnershipFormatError.
+//
+// Windows expects the UserId/GroupId to be in SID format, Linux and Darwin expect it in UID/GID format.
+func SetFileUserGroup(filename, uid, gid string) error {
+	uidI, err := strconv.Atoi(uid)
+	if err != nil && strings.Contains(err.Error(), "") {
+		return err
+	}
+	gidI, err := strconv.Atoi(gid)
+	if err != nil {
+		return err
+	}
+
+	err = os.Lchown(filename, uidI, gidI)
+	if err != nil {
+		return err
+	}
+	return nil
+}

storage/fs_linux.go

@@ -47,3 +47,41 @@ func SetFileTime(filename string, accessTime, modificationTime, creationTime tim
 	}
 	return nil
 }
+
+// GetFileUserGroup will take a filename and return the userId and groupId associated with it.
+//
+//	On windows this is in the format of a SID, on linux/darwin this is in the format of a UID/GID.
+func GetFileUserGroup(filename string) (username, group string, err error) {
+	info, err := os.Stat(filename)
+	if err != nil {
+		return "", "", err
+	}
+
+	stat := info.Sys().(*syscall.Stat_t)
+
+	username = strconv.Itoa(int(stat.Uid))
+	group = strconv.Itoa(int(stat.Gid))
+	return username, group, nil
+}
+
+// SetFileUserGroup will set the UserId and GroupId on a filename.
+//
+//	If the UserId/GroupId format does not match the platform, it will return an InvalidOwnershipFormatError.
+//
+// Windows expects the UserId/GroupId to be in SID format, Linux and Darwin expect it in UID/GID format.
+func SetFileUserGroup(filename, uid, gid string) error {
+	uidI, err := strconv.Atoi(uid)
+	if err != nil {
+		return &InvalidOwnershipFormatError{Err: err}
+	}
+	gidI, err := strconv.Atoi(gid)
+	if err != nil {
+		return &InvalidOwnershipFormatError{Err: err}
+	}
+
+	err = os.Lchown(filename, uidI, gidI)
+	if err != nil {
+		return err
+	}
+	return nil
+}

storage/fs_windows.go

@@ -3,7 +3,10 @@
 package storage
 
 import (
+	"github.com/Microsoft/go-winio"
+	"golang.org/x/sys/windows"
 	"os"
+	"strings"
 	"syscall"
 	"time"
 )
@@ -62,3 +65,85 @@ func SetFileTime(filename string, accessTime, modificationTime, creationTime tim
 	}
 	return nil
 }
+
+// GetFileUserGroup will take a filename and return the userId and groupId associated with it.
+//
+//	On windows this is in the format of a SID, on linux/darwin this is in the format of a UID/GID.
+func GetFileUserGroup(filename string) (userId, groupId string, err error) {
+	sd, err := windows.GetNamedSecurityInfo(filename, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION|windows.GROUP_SECURITY_INFORMATION)
+	if err != nil {
+		return "", "", err
+	}
+
+	userSID, _, err := sd.Owner()
+	groupSID, _, err := sd.Group()
+
+	userId = userSID.String()
+	groupId = groupSID.String()
+
+	return userId, groupId, nil
+}
+
+// SetFileUserGroup will set the UserId and GroupId on a filename.
+//
+//	If the UserId/GroupId format does not match the platform, it will return an InvalidOwnershipFormatError.
+//
+// Windows expects the UserId/GroupId to be in SID format, Linux and Darwin expect it in UID/GID format.
+func SetFileUserGroup(filename, userId, groupId string) error {
+	var err error
+	privileges := []string{"SeRestorePrivilege", "SeTakeOwnershipPrivilege"}
+	if err := winio.EnableProcessPrivileges(privileges); err != nil {
+		return err
+	}
+	defer winio.DisableProcessPrivileges(privileges)
+
+	var uidSid *windows.SID
+	var gidSid *windows.SID
+	if userId != "" {
+		uidSid, err = StringAsSid(userId)
+		if err != nil {
+			return err
+		}
+	}
+
+	if groupId != "" {
+		gidSid, err = StringAsSid(groupId)
+		if err != nil {
+			return err
+		}
+	}
+
+	err = windows.SetNamedSecurityInfo(filename, windows.SE_FILE_OBJECT, windows.OWNER_SECURITY_INFORMATION, uidSid, gidSid, nil, nil)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func StringAsSid(principal string) (*windows.SID, error) {
+	sid, err := windows.StringToSid(principal)
+	if err != nil {
+		if strings.Contains(err.Error(), "The security ID structure is invalid.") {
+			sid, _, _, err = windows.LookupSID("", principal)
+			if err != nil {
+				return nil, &InvalidOwnershipFormatError{Err: err}
+			}
+		} else {
+			return nil, &InvalidOwnershipFormatError{Err: err}
+		}
+	}
+	return sid, nil
+}
+
+func StringSidAsName(strSID string) (name string, err error) {
+	sid, err := StringAsSid(strSID)
+	if err != nil {
+		return "", err
+	}
+	name, _, _, err = sid.LookupAccount("")
+	if err != nil {
+		return "", err
+	}
+	return name, nil
+}

storage/s3.go

@@ -135,6 +135,8 @@ func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error) {
 
 	etag := aws.StringValue(output.ETag)
 	mod := aws.TimeValue(output.LastModified)
+	userId := aws.StringValue(output.Metadata["file-owner"])
+	groupId := aws.StringValue(output.Metadata["file-group"])
 
 	obj := &Object{
 		URL:        url,
@@ -143,6 +145,8 @@ func (s *S3) Stat(ctx context.Context, url *url.URL) (*Object, error) {
 		Size:       aws.Int64Value(output.ContentLength),
 		CreateTime: &time.Time{},
 		AccessTime: &time.Time{},
+		UserId:     userId,
+		GroupId:    groupId,
 	}
 
 	cTimeS := aws.StringValue(output.Metadata["file-ctime"])
@@ -598,6 +602,9 @@ func (s *S3) Copy(ctx context.Context, from, to *url.URL, metadata Metadata) err
 		input.Metadata["file-atime"] = aws.String(atime)
 	}
 
+	input.Metadata["file-owner"] = aws.String(metadata.FileUid)
+	input.Metadata["file-group"] = aws.String(metadata.FileGid)
+
 	if len(metadata.UserDefined) != 0 {
 		m := make(map[string]*string)
 		for k, v := range metadata.UserDefined {