Skip to content

Commit

Permalink
PMM-9288 Extended profiler. (#3662)
Browse files Browse the repository at this point in the history 
* PMM-9288 Extended system profile capture.

* PMM-9288 Tidy.

* PMM-9288 Another fields/metrics.

* PMM-9288 Changes.

* PMM-9288 New fields in clickhouse.

* PMM-9288 Refactor.

* PMM-9288 Changes.

* PMM-9288 Changes.

* PMM-9288 Better debug.

* PMM-9288 Profiler test adjust.

* PMM-9288 Changes.

* PMM-9288 Profiler test.

* PMM-9288 Changes.

* PMM-9288 Mod tidy.

* PMM-9288 Small refactor in naming.

* PMM-9288 Format, lint.

* PMM-9288 Changes after discussion.

* PMM-9288 Format.

* PMM-9288 Mention of new PMM ticket.

* PMM-9288 Related tickets.

* PMM-9288 Changes for FE. Mod.

* PMM-9288 Percona Toolkit merged commit hash.

* PMM-9288 Tidy.
  • Loading branch information
@JiriCtvrtka
JiriCtvrtka authored Mar 7, 2025
1 parent 270a542 commit b1acbff
Show file tree
Hide file tree
Showing 31 changed files with 6,095 additions and 2,484 deletions.
82 changes: 64 additions & 18 deletions agent/agents/mongodb/internal/profiler/aggregator/aggregator.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,7 +35,11 @@ import (

var DefaultInterval = time.Duration(time.Minute)

const reportChanBuffer = 1000
const (
reportChanBuffer = 1000
millisecondsToSeconds = 1000
microsecondsToSeconds = 1000000
)

// New returns configured *Aggregator
func New(timeStart time.Time, agentID string, logger *logrus.Entry, maxQueryLength int32) *Aggregator {
Expand Down Expand Up @@ -258,12 +262,12 @@ func (a *Aggregator) createResult(_ context.Context) *report.Result {
query, truncated := truncate.Query(v.Query, a.maxQueryLength, truncate.GetMongoDBDefaultMaxQueryLength())
bucket := &agentv1.MetricsBucket{
Common: &agentv1.MetricsBucket_Common{
Queryid: v.ID,
Queryid: v.ID, // PMM-13466
Fingerprint: fingerprint,
Database: db,
Tables: []string{collection},
Username: "",
ClientHost: "",
Username: v.User,
ClientHost: v.Client,
AgentId: a.agentID,
AgentType: inventoryv1.AgentType_AGENT_TYPE_QAN_MONGODB_PROFILER_AGENT,
PeriodStartUnixSecs: uint32(a.timeStart.Truncate(1 * time.Minute).Unix()),
Expand All @@ -272,38 +276,80 @@ func (a *Aggregator) createResult(_ context.Context) *report.Result {
ExampleType: agentv1.ExampleType_EXAMPLE_TYPE_RANDOM,
NumQueries: float32(v.Count),
IsTruncated: truncated,
Comments: nil, // PMM-11866
},
Mongodb: &agentv1.MetricsBucket_MongoDB{},
}

bucket.Common.MQueryTimeCnt = float32(v.Count) // TODO: Check is it right value
bucket.Common.MQueryTimeMax = float32(v.QueryTime.Max) / 1000
bucket.Common.MQueryTimeMin = float32(v.QueryTime.Min) / 1000
bucket.Common.MQueryTimeP99 = float32(v.QueryTime.Pct99) / 1000
bucket.Common.MQueryTimeSum = float32(v.QueryTime.Total) / 1000
bucket.Common.MQueryTimeCnt = float32(v.Count) // PMM-13788
bucket.Common.MQueryTimeMax = float32(v.QueryTime.Max) / millisecondsToSeconds
bucket.Common.MQueryTimeMin = float32(v.QueryTime.Min) / millisecondsToSeconds
bucket.Common.MQueryTimeP99 = float32(v.QueryTime.Pct99) / millisecondsToSeconds
bucket.Common.MQueryTimeSum = float32(v.QueryTime.Total) / millisecondsToSeconds

bucket.Mongodb.MDocsReturnedCnt = float32(v.Count) // TODO: Check is it right value
bucket.Mongodb.MDocsReturnedCnt = float32(v.Count) // PMM-13788
bucket.Mongodb.MDocsReturnedMax = float32(v.Returned.Max)
bucket.Mongodb.MDocsReturnedMin = float32(v.Returned.Min)
bucket.Mongodb.MDocsReturnedP99 = float32(v.Returned.Pct99)
bucket.Mongodb.MDocsReturnedSum = float32(v.Returned.Total)

bucket.Mongodb.MDocsScannedCnt = float32(v.Count) // TODO: Check is it right value
bucket.Mongodb.MDocsScannedMax = float32(v.Scanned.Max)
bucket.Mongodb.MDocsScannedMin = float32(v.Scanned.Min)
bucket.Mongodb.MDocsScannedP99 = float32(v.Scanned.Pct99)
bucket.Mongodb.MDocsScannedSum = float32(v.Scanned.Total)

bucket.Mongodb.MResponseLengthCnt = float32(v.Count) // TODO: Check is it right value
bucket.Mongodb.MResponseLengthCnt = float32(v.ResponseLengthCount)
bucket.Mongodb.MResponseLengthMax = float32(v.ResponseLength.Max)
bucket.Mongodb.MResponseLengthMin = float32(v.ResponseLength.Min)
bucket.Mongodb.MResponseLengthP99 = float32(v.ResponseLength.Pct99)
bucket.Mongodb.MResponseLengthSum = float32(v.ResponseLength.Total)

bucket.Mongodb.MFullScanCnt = float32(v.CollScanCount)
bucket.Mongodb.MFullScanSum = float32(v.CollScanSum) / 1000
bucket.Mongodb.MFullScanSum = float32(v.CollScanCount) // Sum is same like count in this case
bucket.Mongodb.PlanSummary = v.PlanSummary

bucket.Mongodb.ApplicationName = v.AppName

bucket.Mongodb.MDocsExaminedCnt = float32(v.DocsExaminedCount)
bucket.Mongodb.MDocsExaminedMax = float32(v.DocsExamined.Max)
bucket.Mongodb.MDocsExaminedMin = float32(v.DocsExamined.Min)
bucket.Mongodb.MDocsExaminedP99 = float32(v.DocsExamined.Pct99)
bucket.Mongodb.MDocsExaminedSum = float32(v.DocsExamined.Total)

bucket.Mongodb.MKeysExaminedCnt = float32(v.KeysExaminedCount)
bucket.Mongodb.MKeysExaminedMax = float32(v.KeysExamined.Max)
bucket.Mongodb.MKeysExaminedMin = float32(v.KeysExamined.Min)
bucket.Mongodb.MKeysExaminedP99 = float32(v.KeysExamined.Pct99)
bucket.Mongodb.MKeysExaminedSum = float32(v.KeysExamined.Total)

bucket.Mongodb.MLocksGlobalAcquireCountReadSharedCnt = float32(v.LocksGlobalAcquireCountReadSharedCount)
bucket.Mongodb.MLocksGlobalAcquireCountReadSharedSum = float32(v.LocksGlobalAcquireCountReadShared)

bucket.Mongodb.MLocksGlobalAcquireCountWriteSharedCnt = float32(v.LocksGlobalAcquireCountWriteSharedCount)
bucket.Mongodb.MLocksGlobalAcquireCountWriteSharedSum = float32(v.LocksGlobalAcquireCountWriteShared)

bucket.Mongodb.MLocksDatabaseAcquireCountReadSharedCnt = float32(v.LocksDatabaseAcquireCountReadSharedCount)
bucket.Mongodb.MLocksDatabaseAcquireCountReadSharedSum = float32(v.LocksDatabaseAcquireCountReadShared)

bucket.Mongodb.MLocksDatabaseAcquireWaitCountReadSharedCnt = float32(v.LocksDatabaseAcquireWaitCountReadSharedCount)
bucket.Mongodb.MLocksDatabaseAcquireWaitCountReadSharedSum = float32(v.LocksDatabaseAcquireWaitCountReadShared)

bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedCnt = float32(v.LocksDatabaseTimeAcquiringMicrosReadSharedCount)
bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedMax = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Max) / microsecondsToSeconds
bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedMin = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Min) / microsecondsToSeconds
bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedP99 = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Pct99) / microsecondsToSeconds
bucket.Mongodb.MLocksDatabaseTimeAcquiringMicrosReadSharedSum = float32(v.LocksDatabaseTimeAcquiringMicrosReadShared.Total) / microsecondsToSeconds

bucket.Mongodb.MLocksCollectionAcquireCountReadSharedCnt = float32(v.LocksCollectionAcquireCountReadSharedCount)
bucket.Mongodb.MLocksCollectionAcquireCountReadSharedSum = float32(v.LocksCollectionAcquireCountReadShared)

bucket.Mongodb.MStorageBytesReadCnt = float32(v.StorageBytesReadCount)
bucket.Mongodb.MStorageBytesReadMax = float32(v.StorageBytesRead.Max)
bucket.Mongodb.MStorageBytesReadMin = float32(v.StorageBytesRead.Min)
bucket.Mongodb.MStorageBytesReadP99 = float32(v.StorageBytesRead.Pct99)
bucket.Mongodb.MStorageBytesReadSum = float32(v.StorageBytesRead.Total)

bucket.Mongodb.MStorageTimeReadingMicrosCnt = float32(v.StorageTimeReadingMicrosCount)
bucket.Mongodb.MStorageTimeReadingMicrosMax = float32(v.StorageTimeReadingMicros.Max) / microsecondsToSeconds
bucket.Mongodb.MStorageTimeReadingMicrosMin = float32(v.StorageTimeReadingMicros.Min) / microsecondsToSeconds
bucket.Mongodb.MStorageTimeReadingMicrosP99 = float32(v.StorageTimeReadingMicros.Pct99) / microsecondsToSeconds
bucket.Mongodb.MStorageTimeReadingMicrosSum = float32(v.StorageTimeReadingMicros.Total) / microsecondsToSeconds

buckets = append(buckets, bucket)
}

Expand Down
48 changes: 30 additions & 18 deletions agent/agents/mongodb/internal/profiler/aggregator/aggregator_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -51,10 +51,11 @@ func TestAggregator(t *testing.T) {
defer aggregator.Stop()
ctx := context.TODO()
err := aggregator.Add(ctx, proto.SystemProfile{
NscannedObjects: 2,
Nreturned: 3,
Ns: "collection.people",
Op: "insert",
Nreturned: 3,
Ns: "collection.people",
Op: "insert",
DocsExamined: 2,
KeysExamined: 3,
})
require.NoError(t, err)

Expand Down Expand Up @@ -85,11 +86,16 @@ func TestAggregator(t *testing.T) {
MDocsReturnedMax: 3,
MDocsReturnedP99: 3,
MResponseLengthCnt: 1,
MDocsScannedCnt: 1,
MDocsScannedSum: 2,
MDocsScannedMin: 2,
MDocsScannedMax: 2,
MDocsScannedP99: 2,
MDocsExaminedCnt: 1,
MDocsExaminedSum: 2,
MDocsExaminedMin: 2,
MDocsExaminedMax: 2,
MDocsExaminedP99: 2,
MKeysExaminedCnt: 1,
MKeysExaminedSum: 3,
MKeysExaminedMin: 3,
MKeysExaminedMax: 3,
MKeysExaminedP99: 3,
},
},
},
Expand All @@ -104,10 +110,9 @@ func TestAggregator(t *testing.T) {
defer aggregator.Stop()
ctx := context.TODO()
err := aggregator.Add(ctx, proto.SystemProfile{
NscannedObjects: 2,
Nreturned: 3,
Ns: "collection.people",
Op: "query",
Nreturned: 3,
Ns: "collection.people",
Op: "query",
Command: bson.D{
primitive.E{Key: "find", Value: "people"},
primitive.E{
Expand All @@ -117,6 +122,8 @@ func TestAggregator(t *testing.T) {
},
},
},
DocsExamined: 2,
KeysExamined: 3,
})
require.NoError(t, err)

Expand Down Expand Up @@ -148,11 +155,16 @@ func TestAggregator(t *testing.T) {
MDocsReturnedMax: 3,
MDocsReturnedP99: 3,
MResponseLengthCnt: 1,
MDocsScannedCnt: 1,
MDocsScannedSum: 2,
MDocsScannedMin: 2,
MDocsScannedMax: 2,
MDocsScannedP99: 2,
MDocsExaminedCnt: 1,
MDocsExaminedSum: 2,
MDocsExaminedMin: 2,
MDocsExaminedMax: 2,
MDocsExaminedP99: 2,
MKeysExaminedCnt: 1,
MKeysExaminedSum: 3,
MKeysExaminedMin: 3,
MKeysExaminedMax: 3,
MKeysExaminedP99: 3,
},
},
},
Expand Down
17 changes: 8 additions & 9 deletions agent/agents/mongodb/internal/profiler/profiler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,7 +149,7 @@ func testProfiler(t *testing.T, url string) {
b.Mongodb.MDocsReturnedCnt += bucket.Mongodb.MDocsReturnedCnt
b.Mongodb.MResponseLengthCnt += bucket.Mongodb.MResponseLengthCnt
b.Mongodb.MResponseLengthSum += bucket.Mongodb.MResponseLengthSum
b.Mongodb.MDocsScannedCnt += bucket.Mongodb.MDocsScannedCnt
b.Mongodb.MDocsExaminedCnt += bucket.Mongodb.MDocsExaminedCnt
} else {
bucketsMap[key] = bucket
}
Expand Down Expand Up @@ -184,16 +184,15 @@ func testProfiler(t *testing.T, url string) {
MResponseLengthMin: responseLength,
MResponseLengthMax: responseLength,
MResponseLengthP99: responseLength,
MDocsScannedCnt: docsCount,
}
// TODO: fix protobuf equality https://jira.percona.com/browse/PMM-6743
assert.Equalf(t, expected.MDocsReturnedCnt, bucket.Mongodb.MDocsReturnedCnt, "wrong metrics for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthCnt, bucket.Mongodb.MResponseLengthCnt, "wrong metrics for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthSum, bucket.Mongodb.MResponseLengthSum, "wrong metrics for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthMin, bucket.Mongodb.MResponseLengthMin, "wrong metrics for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthMax, bucket.Mongodb.MResponseLengthMax, "wrong metrics for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthP99, bucket.Mongodb.MResponseLengthP99, "wrong metrics for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MDocsScannedCnt, bucket.Mongodb.MDocsScannedCnt, "wrong metrics for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MDocsReturnedCnt, bucket.Mongodb.MDocsReturnedCnt, "wrong metrics MDocsReturnedCnt for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthCnt, bucket.Mongodb.MResponseLengthCnt, "wrong metrics MResponseLengthCnt for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthSum, bucket.Mongodb.MResponseLengthSum, "wrong metrics MResponseLengthSum for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthMin, bucket.Mongodb.MResponseLengthMin, "wrong metrics MResponseLengthMin for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthMax, bucket.Mongodb.MResponseLengthMax, "wrong metrics MResponseLengthMax for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MResponseLengthP99, bucket.Mongodb.MResponseLengthP99, "wrong metrics MResponseLengthP99 for db %s", bucket.Common.Database)
assert.Equalf(t, expected.MDocsExaminedCnt, bucket.Mongodb.MDocsExaminedCnt, "wrong metrics MDocsExaminedCnt for db %s", bucket.Common.Database)
}
require.NotNil(t, findBucket)
assert.Equal(t, `db.people.find({"name_00\ufffd":"?"})`, findBucket.Common.Fingerprint)
Expand Down
Loading

0 comments on commit b1acbff

Please sign in to comment.