Skip to content

Commit

Permalink
fix(metric): reintroduce /metrics/vdp/pipeline/triggers endpoint (#243)
Browse files Browse the repository at this point in the history 
Because

- Pipeline metrics list endpoint was removed but it has a client.

This commit

- Reintroduces the endpoint contract. It will be removed with the new
  dashboard design.
  • Loading branch information
@jvallesm
jvallesm authored Oct 23, 2024
1 parent 4c093c6 commit d595d16
Show file tree
Hide file tree
Showing 10 changed files with 622 additions and 246 deletions.
2 changes: 1 addition & 1 deletion go.mod
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ require (
github.com/grpc-ecosystem/grpc-gateway/v2 v2.19.1
github.com/iancoleman/strcase v0.2.0
github.com/influxdata/influxdb-client-go/v2 v2.12.3
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240729070104-1b23ffa897eb
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241023112228-d36cbd2f1d9e
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a
github.com/instill-ai/x v0.4.0-alpha
github.com/knadh/koanf v1.5.0
Expand Down
4 changes: 2 additions & 2 deletions go.sum
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1090,8 +1090,8 @@ github.com/influxdata/influxdb-client-go/v2 v2.12.3 h1:28nRlNMRIV4QbtIUvxhWqaxn0
github.com/influxdata/influxdb-client-go/v2 v2.12.3/go.mod h1:IrrLUbCjjfkmRuaCiGQg4m2GbkaeJDcuWoxiWdQEbA0=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240729070104-1b23ffa897eb h1:3NyTWRr4HDB0Hz83Bd66g7s0qCytHOA7oGL6TVe6UsA=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20240729070104-1b23ffa897eb/go.mod h1:2blmpUwiTwxIDnrjIqT6FhR5ewshZZF554wzjXFvKpQ=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241023112228-d36cbd2f1d9e h1:qUCFv38Xl9Gn9MMAYoHR97i7iQ/L3b8zvj/TX/viZlU=
github.com/instill-ai/protogen-go v0.3.3-alpha.0.20241023112228-d36cbd2f1d9e/go.mod h1:rf0UY7VpEgpaLudYEcjx5rnbuwlBaaLyD4FQmWLtgAY=
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a h1:gmy8BcCFDZQan40c/D3f62DwTYtlCwi0VrSax+pKffw=
github.com/instill-ai/usage-client v0.2.4-alpha.0.20240123081026-6c78d9a5197a/go.mod h1:EpX3Yr661uWULtZf5UnJHfr5rw2PDyX8ku4Kx0UtYFw=
github.com/instill-ai/x v0.4.0-alpha h1:zQV2VLbSHjMv6gyBN/2mwwrvWk0/mJM6ZKS12AzjfQg=
Expand Down
131 changes: 59 additions & 72 deletions integration-test/grpc-public-user.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,84 +239,71 @@ export function CheckPublicGetRemainingCredit(header) {
}

export function CheckPublicMetrics(header) {
group(`Management Public API: List Pipeline Trigger Table Records`, () => {
let pipeline_id = randomString(10);

let pipeline_id = randomString(10)
client.connect(constant.mgmtPublicGRPCHost, {
plaintext: true
});

group(`Management Public API: List Pipeline Trigger Records`, () => {
let emptyPipelineTriggerRecordResponse = {
"pipelineTriggerRecords": [],
"nextPageToken": "",
"totalSize": 0
};

check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords', {}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords response has pipelineTriggerRecords': (r) => r && r.message.pipelineTriggerRecords !== undefined,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords response has totalSize': (r) => r && r.message.totalSize !== undefined,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords response has nextPageToken': (r) => r && r.message.nextPageToken !== undefined,
});

check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords', {
filter: `pipelineId="${pipeline_id}" AND triggerMode=MODE_SYNC`,
}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords with filter status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords with filter response pipelineTriggerRecords length is 0': (r) => r && r.message.pipelineTriggerRecords.length === 0,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords with filter response totalSize is 0': (r) => r && r.message.totalSize === emptyPipelineTriggerRecordResponse.totalSize,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerRecords with filter response nextPageToken is empty': (r) => r && r.message.nextPageToken === emptyPipelineTriggerRecordResponse.nextPageToken,
});
});

group(`Management Public API: List Pipeline Trigger Table Records`, () => {
let emptyPipelineTriggerTableRecordResponse = {
"pipelineTriggerTableRecords": [],
"nextPageToken": "",
"totalSize": 0
}

check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/tables`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response has pipelineTriggerTableRecords`]:
(r) => r.json().pipelineTriggerTableRecords !== undefined,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response has nextPageToken`]:
(r) => r.json().totalSize !== undefined,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables response has totalSize`]:
(r) => r.json().nextPageToken !== undefined,
}
)
check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/tables?filter=pipelineId=%22${pipeline_id}%22`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response pipelineTriggerTableRecords length is 0`]:
(r) => r.json().pipelineTriggerTableRecords.length === 0,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response nextPageToken is empty`]:
(r) => r.json().nextPageToken === emptyPipelineTriggerTableRecordResponse.nextPageToken,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/tables with filter response totalSize is 0`]:
(r) => r.json().totalSize === emptyPipelineTriggerTableRecordResponse.totalSize,
}
)
})
};

check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords', {}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords response has pipelineTriggerTableRecords': (r) => r && r.message.pipelineTriggerTableRecords !== undefined,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords response has totalSize': (r) => r && r.message.totalSize !== undefined,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords response has nextPageToken': (r) => r && r.message.nextPageToken !== undefined,
});
check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords', {
filter: `pipelineId="${pipeline_id}"`,
}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords with filter status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords with filter response pipelineTriggerTableRecords length is 0': (r) => r && r.message.pipelineTriggerTableRecords.length === 0,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords with filter response totalSize is 0': (r) => r && r.message.totalSize === emptyPipelineTriggerTableRecordResponse.totalSize,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerTableRecords with filter response nextPageToken is empty': (r) => r && r.message.nextPageToken === emptyPipelineTriggerTableRecordResponse.nextPageToken,
});
});

group(`Management Public API: List Pipeline Trigger Chart Records`, () => {
check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords', {}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords response has pipelineTriggerChartRecords': (r) => r && r.message.pipelineTriggerChartRecords !== undefined,
});
check(client.invoke('core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords', {
filter: `pipelineId="${pipeline_id}" AND triggerMode=MODE_SYNC`,
}, header), {
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords with filter status': (r) => r && r.status == grpc.StatusOK,
'core.mgmt.v1beta.MgmtPublicService/ListPipelineTriggerChartRecords with filter response pipelineTriggerChartRecords lenght is 0': (r) => r && r.message.pipelineTriggerChartRecords.length === 0,
});
});

let pipeline_id = randomString(10)

check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/charts`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts response has pipelineTriggerRecords`]:
(r) => r.json().pipelineTriggerChartRecords !== undefined,
}
)
check(
http.request(
"GET",
`${constant.mgmtPublicHost}/metrics/vdp/pipeline/charts?filter=triggerMode=MODE_SYNC%20AND%20pipelineId=%22${pipeline_id}%22`,
null,
header,
),
{
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts with filter response status is 200`]:
(r) => r.status === 200,
[`GET /${constant.mgmtVersion}/metrics/vdp/pipeline/charts with filter response pipelineTriggerRecords length is 0`]:
(r) => r.json().pipelineTriggerChartRecords.length === 0,
}
)
})
client.close();
}
175 changes: 111 additions & 64 deletions integration-test/proto/core/mgmt/v1beta/metric.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -39,36 +39,36 @@ message PipelineTriggerCount {
}

/*
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the pipeline UUID.
reserved 2;
// 3 is reserved for the trigger mode. The server wasn't grouping results by this
// field.
reserved 3;
// 4 is reserved for the trigger status. The server wasn't grouping results
// by this field.
reserved 4;
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// 7 is reserved for the trigger execution duration.
reserved 7;
// 8 is reserved for the pipeline release ID. The server wasn't grouping
// results by this field.
reserved 8;
// 9 is reserved for the pipeline release UUID. The server wasn't grouping
// results by this field.
reserved 9;
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}
// PipelineTriggerChartRecord represents a timeline of pipeline triggers. It
// contains a collection of (timestamp, count) pairs that represent the total
// pipeline triggers in a given time bucket.
// pipeline ID and time frame.
message PipelineTriggerChartRecord {
// This field will be present present when the information is grouped by pipeline.
optional string pipeline_id = 1 [(google.api.field_behavior) = OUTPUT_ONLY];
// 2 is reserved for the pipeline UUID.
reserved 2;
// 3 is reserved for the trigger mode. The server wasn't grouping results by this
// field.
reserved 3;
// 4 is reserved for the trigger status. The server wasn't grouping results
// by this field.
reserved 4;
// Time buckets.
repeated google.protobuf.Timestamp time_buckets = 5 [(google.api.field_behavior) = OUTPUT_ONLY];
// Aggregated trigger count in each time bucket.
repeated int32 trigger_counts = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// 7 is reserved for the trigger execution duration.
reserved 7;
// 8 is reserved for the pipeline release ID. The server wasn't grouping
// results by this field.
reserved 8;
// 9 is reserved for the pipeline release UUID. The server wasn't grouping
// results by this field.
reserved 9;
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 10 [(google.api.field_behavior) = OUTPUT_ONLY];
}
*/

// GetPipelineTriggerCountRequest represents a request to fetch the trigger
Expand All @@ -92,42 +92,42 @@ message GetPipelineTriggerCountResponse {
}

/*
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// 1 is reserved for the aggregation window in nanoseconds. This is
// deprecated in favour of an aggregation window string that represents a
// duration.
reserved 1;
// 2 is reserved for the filter. For now, this endpoint won't allow filtering
// but in the future we might implement a filter to show the trigger count of
// only certain pipelines and to group by the pipeline ID.
reserved 2;
// ListPipelineTriggerChartRecordsRequest represents a request to list pipeline
// trigger chart records for a given requester, grouped by time buckets.
message ListPipelineTriggerChartRecordsRequest {
// 1 is reserved for the aggregation window in nanoseconds. This is
// deprecated in favour of an aggregation window string that represents a
// duration.
reserved 1;
// 2 is reserved for the filter. For now, this endpoint won't allow filtering
// but in the future we might implement a filter to show the trigger count of
// only certain pipelines and to group by the pipeline ID.
reserved 2;
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 3 [(google.api.field_behavior) = REQUIRED];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 4;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 5;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 6;
}
// The ID of the namespace that requested the pipeline triggers.
string namespace_id = 3 [(google.api.field_behavior) = REQUIRED];
// Aggregation window. The value is a positive duration string, i.e. a
// sequence of decimal numbers, each with optional fraction and a unit
// suffix, such as "300ms", "1.5h" or "2h45m".
// The minimum (and default) window is 1h.
optional string aggregation_window = 4;
// Beginning of the time range from which the records will be fetched.
// The default value is the beginning of the current day, in UTC.
optional google.protobuf.Timestamp start = 5;
// End of the time range from which the records will be fetched.
// The default value is the current timestamp.
optional google.protobuf.Timestamp stop = 6;
}
// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}
// ListPipelineTriggerChartRecordsResponse contains a list of pipeline trigger
// chart records.
message ListPipelineTriggerChartRecordsResponse {
// Pipeline trigger counts. Until we allow filtering or grouping by fields
// like pipeline ID, this list will contain only one element with the
// timeline of trigger counts for a given requester, regardless the pipeline
// ID, trigger mode, final status or other fields.
repeated PipelineTriggerChartRecord pipeline_trigger_chart_records = 1;
}
*/

// CreditConsumptionChartRecord represents a timeline of Instill Credit
Expand Down Expand Up @@ -258,3 +258,50 @@ message PipelineTriggerChartRecord {
// Release UUID for the triggered pipeline if it is a release pipeline.
string pipeline_release_uid = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// PipelineTriggerRecord represents a pipeline execution event.
message PipelineTriggerRecord {
// The moment when the pipeline was triggered.
google.protobuf.Timestamp trigger_time = 1;
// UUID of the trigger.
string pipeline_trigger_id = 2;
// Pipeline ID.
string pipeline_id = 3;
// Pipeline UUID.
string pipeline_uid = 4;
// Trigger mode.
Mode trigger_mode = 5;
// Total execution duration.
float compute_time_duration = 6 [(google.api.field_behavior) = OUTPUT_ONLY];
// Final status.
Status status = 7 [(google.api.field_behavior) = OUTPUT_ONLY];
// If a release of the pipeline was triggered, pipeline version.
string pipeline_release_id = 8 [(google.api.field_behavior) = OUTPUT_ONLY];
// If a release of the pipeline was triggered, release UUID.
string pipeline_release_uid = 9 [(google.api.field_behavior) = OUTPUT_ONLY];
}

// ListPipelineTriggerRecordsRequest represents a request to list the triggers
// of a pipeline.
message ListPipelineTriggerRecordsRequest {
// The maximum number of triggers to return. If this parameter is unspecified,
// at most 100 pipelines will be returned. The cap value for this parameter is
// 1000 (i.e. any value above that will be coerced to 100).
optional int32 page_size = 1 [(google.api.field_behavior) = OPTIONAL];
// Page token.
optional string page_token = 2 [(google.api.field_behavior) = OPTIONAL];
// Filter can hold an [AIP-160](https://google.aip.dev/160)-compliant filter
// expression.
// - Example: `create_time>timestamp("2000-06-19T23:31:08.657Z")`.
optional string filter = 3 [(google.api.field_behavior) = OPTIONAL];
}

// ListPipelineTriggerRecordsResponse contains a list of pipeline triggers.
message ListPipelineTriggerRecordsResponse {
// A list of pipeline triggers.
repeated PipelineTriggerRecord pipeline_trigger_records = 1;
// Next page token.
string next_page_token = 2;
// Total number of pipeline triggers.
int32 total_size = 3;
}
Loading

0 comments on commit d595d16

Please sign in to comment.