Merge branch 'dev' of https://github.com/percona/percona-backup-mongodb…

… into PBM-1474-aws-s3-migration

.github/workflows/ci.yml

@@ -44,7 +44,7 @@ jobs:
         psmdb: ["6.0", "7.0", "8.0"]
         test: [logical, physical, incremental, external]
     env:
-      PBM_BRANCH: ${{ github.event.inputs.pbm_branch || 'main' }}
+      PBM_BRANCH: ${{ github.event.inputs.pbm_branch || github.ref_name }}
       GO_VER: ${{ github.event.inputs.go_ver || '1.22-bullseye' }}
       PR_NUMBER: ${{ github.event.number|| github.event.inputs.pr_ver }}
       MAKE_TARGET: 'build-cover'

cmd/pbm-agent/main.go

@@ -59,6 +59,8 @@ func rootCommand() *cobra.Command {
 		},
 	}
 
+	rootCmd.CompletionOptions.DisableDefaultCmd = true
+
 	setRootFlags(rootCmd)
 	return rootCmd
 }

cmd/pbm-agent/restore.go

@@ -87,6 +87,31 @@ func (a *Agent) Restore(ctx context.Context, r *ctrl.RestoreCmd, opid ctrl.OPID,
 		a.removePitr()
 	}
 
+	// stop balancer during the restore
+	if a.brief.Sharded && nodeInfo.IsClusterLeader() {
+		bs, err := topo.GetBalancerStatus(ctx, a.leadConn)
+		if err != nil {
+			l.Error("get balancer status: %v", err)
+			return
+		}
+
+		if bs.IsOn() {
+			err := topo.SetBalancerStatus(ctx, a.leadConn, topo.BalancerModeOff)
+			if err != nil {
+				l.Error("set balancer off: %v", err)
+			}
+
+			l.Debug("waiting for balancer off")
+			bs := topo.WaitForBalancerDisabled(ctx, a.leadConn, time.Second*30, l)
+			if bs.IsDisabled() {
+				l.Debug("balancer is disabled")
+			} else {
+				l.Warning("balancer is not disabled: balancer mode: %s, in balancer round: %t",
+					bs.Mode, bs.InBalancerRound)
+			}
+		}
+	}
+
 	var bcpType defs.BackupType
 	var bcp *backup.BackupMeta
 

cmd/pbm/main.go

@@ -117,6 +117,8 @@ func newPbmApp() *pbmApp {
 		SilenceUsage:       true,
 	}
 
+	app.rootCmd.CompletionOptions.DisableDefaultCmd = true
+
 	app.rootCmd.PersistentFlags().String(
 		mongoConnFlag,
 		"",
@@ -385,6 +387,10 @@ func (app *pbmApp) buildConfigCmd() *cobra.Command {
 
 func (app *pbmApp) buildConfigProfileCmd() *cobra.Command {
 	runListConfigProfileCmd := app.wrapRunE(func(cmd *cobra.Command, args []string) (fmt.Stringer, error) {
+		if len(args) != 0 {
+			return nil, errors.New(fmt.Sprintf("parse command line parameters: unexpected %s", args[0]))
+		}
+
 		return handleListConfigProfiles(app.ctx, app.pbm)
 	})
 

go.mod

@@ -27,7 +27,7 @@ require (
 	github.com/testcontainers/testcontainers-go/modules/mongodb v0.34.0
 	go.mongodb.org/mongo-driver v1.17.1
 	golang.org/x/mod v0.19.0
-	golang.org/x/sync v0.10.0
+	golang.org/x/sync v0.11.0
 	gopkg.in/yaml.v2 v2.4.0
 )
 
@@ -106,12 +106,12 @@ require (
 	go.opentelemetry.io/otel/trace v1.24.0 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
-	golang.org/x/crypto v0.31.0 // indirect
+	golang.org/x/crypto v0.33.0 // indirect
 	golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/sys v0.28.0 // indirect
-	golang.org/x/term v0.27.0 // indirect
-	golang.org/x/text v0.21.0 // indirect
+	golang.org/x/net v0.35.0 // indirect
+	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/term v0.29.0 // indirect
+	golang.org/x/text v0.22.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

go.sum

@@ -263,8 +263,8 @@ golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACk
 golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI=
 golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
+golang.org/x/crypto v0.33.0 h1:IOBPskki6Lysi0lo9qQvbxiQ+FvsCC/YWOecCHAixus=
+golang.org/x/crypto v0.33.0/go.mod h1:bVdXmD7IV/4GdElGPozy6U7lWdRXA4qyRVGJV57uQ5M=
 golang.org/x/exp v0.0.0-20240529005216-23cca8864a10 h1:vpzMC/iZhYFAjJzHU0Cfuq+w1vLLsF2vLkDrPjzKYck=
 golang.org/x/exp v0.0.0-20240529005216-23cca8864a10/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc=
 golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA=
@@ -278,14 +278,14 @@ golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU=
 golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
 golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c=
-golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ=
-golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE=
+golang.org/x/net v0.35.0 h1:T5GQRQb2y08kTAByq9L4/bz8cipCdA8FbRTXewonqY8=
+golang.org/x/net v0.35.0/go.mod h1:EglIi67kWsHKlRzzVMUD93VMSWGFOMSZgxFjparz1Qk=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ=
-golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
+golang.org/x/sync v0.11.0 h1:GGz8+XQP4FvTTrjZPzNKTMFtSXH80RAzG+5ghFPgK9w=
+golang.org/x/sync v0.11.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
@@ -301,18 +301,18 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
-golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q=
-golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM=
+golang.org/x/term v0.29.0 h1:L6pJp37ocefwRRtYPKSWOWzOtWSxVajvz2ldH/xi3iU=
+golang.org/x/term v0.29.0/go.mod h1:6bl4lRlvVuDgSf3179VpIxBF0o10JUpXWOnI7nErv7s=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
+golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
+golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
 golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk=
 golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=

packaging/rpm/mongodb-backup.spec

@@ -4,6 +4,7 @@
 %{?el7:          %global systemd 1}
 %{?el8:          %global systemd 1}
 %{?el9:          %global systemd 1}
+%{?amzn2023:     %global systemd 1}
 
 
 Name:  percona-backup-mongodb

packaging/rpm/pbm-agent.init

@@ -15,7 +15,7 @@ OPTIONS="--mongodb-uri='${PBM_MONGODB_URI}'"
 PIDFILEPATH=/run/pbm-agent/pbm-agent.pid
 PIDDIR=$(dirname $PIDFILEPATH)
 USER=mongod
-GROUP=mogod
+GROUP=mongod
 pbm_agent=${PBM_AGENT:-/usr/bin/pbm-agent}
 pbm_log="/var/log/pbm-agent.log"
 

pbm/archive/backup.go

@@ -31,6 +31,32 @@ var RouterConfigCollections = []string{
 
 const MetaFileV2 = "meta.pbm"
 
+var validIndexOptions = map[string]bool{
+	"2dsphereIndexVersion":    true,
+	"background":              true,
+	"bits":                    true,
+	"bucketSize":              true,
+	"coarsestIndexedLevel":    true,
+	"collation":               true,
+	"default_language":        true,
+	"expireAfterSeconds":      true,
+	"finestIndexedLevel":      true,
+	"key":                     true,
+	"language_override":       true,
+	"max":                     true,
+	"min":                     true,
+	"name":                    true,
+	"ns":                      true,
+	"partialFilterExpression": true,
+	"sparse":                  true,
+	"storageEngine":           true,
+	"textIndexVersion":        true,
+	"unique":                  true,
+	"v":                       true,
+	"weights":                 true,
+	"wildcardProjection":      true,
+}
+
 type ArchiveMetaV2 struct {
 	Version       string `bson:"version"`
 	ServerVersion string `bson:"serverVersion"`
@@ -46,7 +72,7 @@ type NamespaceV2 struct {
 	UUID    string `bson:"uuid,omitempty"`
 	Options bson.D `bson:"options,omitempty"`
 
-	Indexes []*IndexSpec `bson:"indexes"`
+	Indexes []IndexSpec `bson:"indexes"`
 
 	CRC  int64 `bson:"crc"`
 	Size int64 `bson:"size"`
@@ -63,18 +89,7 @@ func (s *NamespaceV2) IsTimeseries() bool { return s.Type == "timeseries" }
 func (s *NamespaceV2) IsSystemCollection() bool { return strings.HasPrefix(s.Name, "system.") }
 func (s *NamespaceV2) IsBucketCollection() bool { return strings.HasPrefix(s.Name, "system.buckets") }
 
-type IndexSpec struct {
-	Version int32 `bson:"v"`
-
-	Key  bson.D `bson:"key"`
-	Name string `bson:"name"`
-
-	Sparse    *bool `bson:"sparse,omitempty"`
-	Unique    *bool `bson:"unique,omitempty"`
-	Clustered *bool `bson:"clustered,omitempty"`
-
-	ExpireAfterSeconds *int32 `bson:"expireAfterSeconds,omitempty"`
-}
+type IndexSpec bson.D
 
 type BackupOptions struct {
 	Client  *mongo.Client
@@ -236,7 +251,7 @@ func (bcp *backupImpl) listDBNamespaces(ctx context.Context, db string) ([]*Name
 				return nil, errors.Wrapf(err, "list indexes for %s", ns.Name)
 			}
 		} else {
-			ns.Indexes = []*IndexSpec{}
+			ns.Indexes = []IndexSpec{}
 		}
 
 		rv = append(rv, ns)
@@ -245,15 +260,33 @@ func (bcp *backupImpl) listDBNamespaces(ctx context.Context, db string) ([]*Name
 	return rv, errors.Wrap(cur.Err(), "cursor")
 }
 
-func (bcp *backupImpl) listIndexes(ctx context.Context, db, coll string) ([]*IndexSpec, error) {
+// listIndexes fetches index definitions for the specified namespace.
+// It returns dynamic bson.D index representation which is filtered by allowed index
+// specification keys.
+func (bcp *backupImpl) listIndexes(ctx context.Context, db, coll string) ([]IndexSpec, error) {
 	cur, err := bcp.conn.Database(db).Collection(coll).Indexes().List(ctx)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "listIndexes cmd for ns: %s.%s", db, coll)
+	}
+
+	idxs := []bson.D{}
+	err = cur.All(ctx, &idxs)
+	if err != nil {
+		return nil, errors.Wrapf(err, "decode indexes for ns: %s.%s", db, coll)
+	}
+
+	var idxSpecs []IndexSpec
+	for i := range idxs {
+		var idxSpec IndexSpec
+		for _, opt := range idxs[i] {
+			if _, ok := validIndexOptions[opt.Key]; ok {
+				idxSpec = append(idxSpec, opt)
+			}
+		}
+		idxSpecs = append(idxSpecs, idxSpec)
 	}
 
-	var indexes []*IndexSpec
-	err = cur.All(ctx, &indexes)
-	return indexes, err
+	return idxSpecs, nil
 }
 
 func (bcp *backupImpl) dumpAllCollections(ctx context.Context, nss []*NamespaceV2) error {