Merge branch 'v3' into PMM-13171-add-clickhouse-credentials

percona · Mar 10, 2025 · 09b9e16 · 09b9e16
2 parents 1d360a1 + 2a5099f
commit 09b9e16
Show file tree

Hide file tree

Showing 44 changed files with 6,148 additions and 3,223 deletions.
diff --git a/.github/workflows/admin.yml b/.github/workflows/admin.yml
@@ -46,7 +46,7 @@ jobs:
           cache: false
 
       - name: Enable Go build cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-go-build-${{ github.ref }}-${{ hashFiles('**') }}
@@ -55,7 +55,7 @@ jobs:
             ${{ runner.os }}-go-build-
 
       - name: Enable Go modules cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }}
@@ -109,7 +109,7 @@ jobs:
           cache: false
 
       - name: Enable Go build cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-go-build-${{ github.ref }}-${{ hashFiles('**') }}
@@ -118,7 +118,7 @@ jobs:
             ${{ runner.os }}-go-build-
 
       - name: Enable Go modules cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }}
@@ -158,7 +158,7 @@ jobs:
 
       - name: Attach the report on failure
         if: failure()
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: "report-${{ matrix.go-version }}-${{ matrix.test-type }}"
           path: ${{ github.workspace }}/cli-tests/playwright-report/

diff --git a/.github/workflows/agent.yml b/.github/workflows/agent.yml
@@ -83,7 +83,7 @@ jobs:
           cache: false
 
       - name: Enable Go build cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-go-build-${{ github.ref }}-${{ hashFiles('**') }}
@@ -92,7 +92,7 @@ jobs:
             ${{ runner.os }}-go-build-
 
       - name: Enable Go modules cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }}

diff --git a/.github/workflows/api-tests.yml b/.github/workflows/api-tests.yml
@@ -146,7 +146,7 @@ jobs:
 
       - name: Upload the logs on failure
         if: ${{ failure() }}
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v4.6.1
         with:
           name: "logs.zip"
           path: ${{ github.workspace }}/logs.zip

diff --git a/.github/workflows/clean.yml b/.github/workflows/clean.yml
@@ -45,15 +45,15 @@ jobs:
           lfs: true
 
       - name: Enable Go modules cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/go/pkg/mod
           key: ${{ matrix.go.version }}-modules-${{ hashFiles('**/go.sum') }}
           restore-keys: |
             ${{ matrix.go.version }}-modules-
 
       - name: Enable Go build cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/go-build
           key: ${{ matrix.go.version }}-build-${{ github.ref }}-${{ hashFiles('**') }}

diff --git a/.github/workflows/devcontainer.yml b/.github/workflows/devcontainer.yml
@@ -54,7 +54,7 @@ jobs:
           password: ${{ secrets.DOCKERHUB_PASSWORD }}
 
       - name: Build and push to registries
-        uses: docker/build-push-action@0adf9959216b96bec444f325f1e493d4aa344497 # v6.14.0
+        uses: docker/build-push-action@471d1dc4e07e5cdedd4c2171150001c434f0b7a4 # v6.15.0
         with:
           file: ./.devcontainer/Dockerfile
           context: .

diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml
@@ -30,7 +30,7 @@ jobs:
           cache: false
 
       - name: Enable Go build cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-go-build-${{ github.ref }}-${{ hashFiles('**') }}
@@ -39,7 +39,7 @@ jobs:
             ${{ runner.os }}-go-build-
 
       - name: Enable Go modules cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }}

diff --git a/.github/workflows/qan-api2.yml b/.github/workflows/qan-api2.yml
@@ -47,7 +47,7 @@ jobs:
           cache: false
 
       - name: Enable Go build cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-go-build-${{ github.ref }}-${{ hashFiles('**') }}
@@ -56,7 +56,7 @@ jobs:
             ${{ runner.os }}-go-build-
 
       - name: Enable Go modules cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }}

diff --git a/.github/workflows/scorecard.yml b/.github/workflows/scorecard.yml
@@ -28,21 +28,21 @@ jobs:
           persist-credentials: false
 
       - name: Run analysis
-        uses: ossf/scorecard-action@62b2cac7ed8198b15735ed49ab1e5cf35480ba46 # v2.4.0
+        uses: ossf/scorecard-action@f49aabe0b5af0936a0987cfb85d86b75731b0186 # v2.4.1
         with:
           results_file: results.sarif
           results_format: sarif
           publish_results: true
 
       - name: Upload results
-        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v3.pre.node20
+        uses: actions/upload-artifact@4cec3d8aa04e39d1a68397de0c4cd6fb9dce8ec1 # v3.pre.node20
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard (optional).
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@9e8d0789d4a0fa9ceb6b1738f7e269594bdd67f0 # v3.28.9
+        uses: github/codeql-action/upload-sarif@b56ba49b26e50535fa1e7f7db0f4f7b4bf65d80d # v3.28.10
         with:
           sarif_file: results.sarif
diff --git a/.github/workflows/vmproxy.yml b/.github/workflows/vmproxy.yml
@@ -47,7 +47,7 @@ jobs:
           cache: false
 
       - name: Enable Go build cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/.cache/go-build
           key: ${{ runner.os }}-go-build-${{ github.ref }}-${{ hashFiles('**') }}
@@ -56,7 +56,7 @@ jobs:
             ${{ runner.os }}-go-build-
 
       - name: Enable Go modules cache
-        uses: actions/cache@0c907a75c2c80ebcb7f088228285e798b750cf8f # v4.2.1
+        uses: actions/cache@d4323d4df104b026a6aa633fdb11d772146be0bf # v4.2.2
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-modules-${{ hashFiles('**/go.sum') }}

diff --git a/agent/agents/mongodb/internal/profiler/aggregator/aggregator.go b/agent/agents/mongodb/internal/profiler/aggregator/aggregator.go
@@ -35,7 +35,11 @@ import (
 
 var DefaultInterval = time.Duration(time.Minute)
 
-const reportChanBuffer = 1000
+const (
+	reportChanBuffer      = 1000
+	millisecondsToSeconds = 1000
+	microsecondsToSeconds = 1000000
+)
 
 // New returns configured *Aggregator
 func New(timeStart time.Time, agentID string, logger *logrus.Entry, maxQueryLength int32) *Aggregator {
@@ -258,12 +262,12 @@ func (a *Aggregator) createResult(_ context.Context) *report.Result {
 		query, truncated := truncate.Query(v.Query, a.maxQueryLength, truncate.GetMongoDBDefaultMaxQueryLength())
 		bucket := &agentv1.MetricsBucket{
 			Common: &agentv1.MetricsBucket_Common{
-				Queryid:             v.ID,
+				Queryid:             v.ID, // PMM-13466
 				Fingerprint:         fingerprint,
 				Database:            db,
 				Tables:              []string{collection},
-				Username:            "",
-				ClientHost:          "",
+				Username:            v.User,
+				ClientHost:          v.Client,
 				AgentId:             a.agentID,
 				AgentType:           inventoryv1.AgentType_AGENT_TYPE_QAN_MONGODB_PROFILER_AGENT,
 				PeriodStartUnixSecs: uint32(a.timeStart.Truncate(1 * time.Minute).Unix()),
@@ -272,38 +276,80 @@ func (a *Aggregator) createResult(_ context.Context) *report.Result {
 				ExampleType:         agentv1.ExampleType_EXAMPLE_TYPE_RANDOM,
 				NumQueries:          float32(v.Count),
 				IsTruncated:         truncated,
+				Comments:            nil, // PMM-11866
 			},
 			Mongodb: &agentv1.MetricsBucket_MongoDB{},
 		}
 
-		bucket.Common.MQueryTimeCnt = float32(v.Count) // TODO: Check is it right value
-		bucket.Common.MQueryTimeMax = float32(v.QueryTime.Max) / 1000
-		bucket.Common.MQueryTimeMin = float32(v.QueryTime.Min) / 1000
-		bucket.Common.MQueryTimeP99 = float32(v.QueryTime.Pct99) / 1000
-		bucket.Common.MQueryTimeSum = float32(v.QueryTime.Total) / 1000
+		bucket.Common.MQueryTimeCnt = float32(v.Count) // PMM-13788
+		bucket.Common.MQueryTimeMax = float32(v.QueryTime.Max) / millisecondsToSeconds
+		bucket.Common.MQueryTimeMin = float32(v.QueryTime.Min) / millisecondsToSeconds
+		bucket.Common.MQueryTimeP99 = float32(v.QueryTime.Pct99) / millisecondsToSeconds
+		bucket.Common.MQueryTimeSum = float32(v.QueryTime.Total) / millisecondsToSeconds
 
-		bucket.Mongodb.MDocsReturnedCnt = float32(v.Count) // TODO: Check is it right value
+		bucket.Mongodb.MDocsReturnedCnt = float32(v.Count) // PMM-13788
 		bucket.Mongodb.MDocsReturnedMax = float32(v.Returned.Max)
 		bucket.Mongodb.MDocsReturnedMin = float32(v.Returned.Min)
 		bucket.Mongodb.MDocsReturnedP99 = float32(v.Returned.Pct99)
 		bucket.Mongodb.MDocsReturnedSum = float32(v.Returned.Total)
 
-		bucket.Mongodb.MDocsScannedCnt = float32(v.Count) // TODO: Check is it right value
-		bucket.Mongodb.MDocsScannedMax = float32(v.Scanned.Max)
-		bucket.Mongodb.MDocsScannedMin = float32(v.Scanned.Min)
-		bucket.Mongodb.MDocsScannedP99 = float32(v.Scanned.Pct99)
-		bucket.Mongodb.MDocsScannedSum = float32(v.Scanned.Total)
-
-		bucket.Mongodb.MResponseLengthCnt = float32(v.Count) // TODO: Check is it right value
+		bucket.Mongodb.MResponseLengthCnt = float32(v.ResponseLengthCount)
 		bucket.Mongodb.MResponseLengthMax = float32(v.ResponseLength.Max)
 		bucket.Mongodb.MResponseLengthMin = float32(v.ResponseLength.Min)
 		bucket.Mongodb.MResponseLengthP99 = float32(v.ResponseLength.Pct99)
 		bucket.Mongodb.MResponseLengthSum = float32(v.ResponseLength.Total)
 
 		bucket.Mongodb.MFullScanCnt = float32(v.CollScanCount)
-		bucket.Mongodb.MFullScanSum = float32(v.CollScanSum) / 1000
+		bucket.Mongodb.MFullScanSum = float32(v.CollScanCount) // Sum is same like count in this case
 		bucket.Mongodb.PlanSummary = v.PlanSummary
 
+		bucket.Mongodb.ApplicationName = v.AppName
+
+		bucket.Mongodb.MDocsExaminedCnt = float32(v.DocsExaminedCount)
+		bucket.Mongodb.MDocsExaminedMax = float32(v.DocsExamined.Max)
+		bucket.Mongodb.MDocsExaminedMin = float32(v.DocsExamined.Min)
+		bucket.Mongodb.MDocsExaminedP99 = float32(v.DocsExamined.Pct99)
+		bucket.Mongodb.MDocsExaminedSum = float32(v.DocsExamined.Total)
+
+		bucket.Mongodb.MKeysExaminedCnt = float32(v.KeysExaminedCount)
+		bucket.Mongodb.MKeysExaminedMax = float32(v.KeysExamined.Max)
+		bucket.Mongodb.MKeysExaminedMin = float32(v.KeysExamined.Min)
+		bucket.Mongodb.MKeysExaminedP99 = float32(v.KeysExamined.Pct99)
+		bucket.Mongodb.MKeysExaminedSum = float32(v.KeysExamined.Total)
+
+		bucket.Mongodb.MLocksGlobalAcquireCountReadSharedCnt = float32(v.LocksGlobalAcquireCountReadSharedCount)
+		bucket.Mongodb.MLocksGlobalAcquireCountReadSharedSum = float32(v.LocksGlobalAcquireCountReadShared)
+
+		bucket.Mongodb.MLocksGlobalAcquireCountWriteSharedCnt = float32(v.LocksGlobalAcquireCountWriteSharedCount)
+		bucket.Mongodb.MLocksGlobalAcquireCountWriteSharedSum = float32(v.LocksGlobalAcquireCountWriteShared)
+
+		bucket.Mongodb.MLocksDatabaseAcquireCountReadSharedCnt = float32(v.LocksDatabaseAcquireCountReadSharedCount)
+		bucket.Mongodb.MLocksDatabaseAcquireCountReadSharedSum = float32(v.LocksDatabaseAcquireCountReadShared)
+
+		bucket.Mongodb.MLocksDatabaseAcquireWaitCountReadSharedCnt = float32(v.LocksDatabaseAcquireWaitCountReadSharedCount)
+		bucket.Mongodb.MLocksDatabaseAcquireWaitCountReadSharedSum = float32(v.LocksDatabaseAcquireWaitCountReadShared)
+
+		bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedCnt = float32(v.LocksDatabaseTimeAcquiringMicrosReadSharedCount)
+		bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedMax = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Max) / microsecondsToSeconds
+		bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedMin = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Min) / microsecondsToSeconds
+		bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedP99 = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Pct99) / microsecondsToSeconds
+		bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedSum = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Total) / microsecondsToSeconds
+
+		bucket.Mongodb.MLocksCollectionAcquireCountReadSharedCnt = float32(v.LocksCollectionAcquireCountReadSharedCount)
+		bucket.Mongodb.MLocksCollectionAcquireCountReadSharedSum = float32(v.LocksCollectionAcquireCountReadShared)
+
+		bucket.Mongodb.MStorageBytesReadCnt = float32(v.StorageBytesReadCount)
+		bucket.Mongodb.MStorageBytesReadMax = float32(v.StorageBytesRead.Max)
+		bucket.Mongodb.MStorageBytesReadMin = float32(v.StorageBytesRead.Min)
+		bucket.Mongodb.MStorageBytesReadP99 = float32(v.StorageBytesRead.Pct99)
+		bucket.Mongodb.MStorageBytesReadSum = float32(v.StorageBytesRead.Total)
+
+		bucket.Mongodb.MStorageTimeReadingMicrosCnt = float32(v.StorageTimeReadingMicrosCount)
+		bucket.Mongodb.MStorageTimeReadingMicrosMax = float32(v.StorageTimeReadingMicros.Max) / microsecondsToSeconds
+		bucket.Mongodb.MStorageTimeReadingMicrosMin = float32(v.StorageTimeReadingMicros.Min) / microsecondsToSeconds
+		bucket.Mongodb.MStorageTimeReadingMicrosP99 = float32(v.StorageTimeReadingMicros.Pct99) / microsecondsToSeconds
+		bucket.Mongodb.MStorageTimeReadingMicrosSum = float32(v.StorageTimeReadingMicros.Total) / microsecondsToSeconds
+
 		buckets = append(buckets, bucket)
 	}
 

diff --git a/agent/agents/mongodb/internal/profiler/aggregator/aggregator_test.go b/agent/agents/mongodb/internal/profiler/aggregator/aggregator_test.go
@@ -51,10 +51,11 @@ func TestAggregator(t *testing.T) {
 		defer aggregator.Stop()
 		ctx := context.TODO()
 		err := aggregator.Add(ctx, proto.SystemProfile{
-			NscannedObjects: 2,
-			Nreturned:       3,
-			Ns:              "collection.people",
-			Op:              "insert",
+			Nreturned:    3,
+			Ns:           "collection.people",
+			Op:           "insert",
+			DocsExamined: 2,
+			KeysExamined: 3,
 		})
 		require.NoError(t, err)
 
@@ -85,11 +86,16 @@ func TestAggregator(t *testing.T) {
 						MDocsReturnedMax:   3,
 						MDocsReturnedP99:   3,
 						MResponseLengthCnt: 1,
-						MDocsScannedCnt:    1,
-						MDocsScannedSum:    2,
-						MDocsScannedMin:    2,
-						MDocsScannedMax:    2,
-						MDocsScannedP99:    2,
+						MDocsExaminedCnt:   1,
+						MDocsExaminedSum:   2,
+						MDocsExaminedMin:   2,
+						MDocsExaminedMax:   2,
+						MDocsExaminedP99:   2,
+						MKeysExaminedCnt:   1,
+						MKeysExaminedSum:   3,
+						MKeysExaminedMin:   3,
+						MKeysExaminedMax:   3,
+						MKeysExaminedP99:   3,
 					},
 				},
 			},
@@ -104,10 +110,9 @@ func TestAggregator(t *testing.T) {
 		defer aggregator.Stop()
 		ctx := context.TODO()
 		err := aggregator.Add(ctx, proto.SystemProfile{
-			NscannedObjects: 2,
-			Nreturned:       3,
-			Ns:              "collection.people",
-			Op:              "query",
+			Nreturned: 3,
+			Ns:        "collection.people",
+			Op:        "query",
 			Command: bson.D{
 				primitive.E{Key: "find", Value: "people"},
 				primitive.E{
@@ -117,6 +122,8 @@ func TestAggregator(t *testing.T) {
 					},
 				},
 			},
+			DocsExamined: 2,
+			KeysExamined: 3,
 		})
 		require.NoError(t, err)
 
@@ -148,11 +155,16 @@ func TestAggregator(t *testing.T) {
 						MDocsReturnedMax:   3,
 						MDocsReturnedP99:   3,
 						MResponseLengthCnt: 1,
-						MDocsScannedCnt:    1,
-						MDocsScannedSum:    2,
-						MDocsScannedMin:    2,
-						MDocsScannedMax:    2,
-						MDocsScannedP99:    2,
+						MDocsExaminedCnt:   1,
+						MDocsExaminedSum:   2,
+						MDocsExaminedMin:   2,
+						MDocsExaminedMax:   2,
+						MDocsExaminedP99:   2,
+						MKeysExaminedCnt:   1,
+						MKeysExaminedSum:   3,
+						MKeysExaminedMin:   3,
+						MKeysExaminedMax:   3,
+						MKeysExaminedP99:   3,
 					},
 				},
 			},