Add otel metrics (opensearch-project#1314)

* List all available otel docs Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Upgraded Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * fetching bin count Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Histogram working Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Cleaned up code Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Viz working with real time data Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Successfully saving to dashboards Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Fixed sidebar jest test Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * View otel metric in dashboards Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * update snapshot and remove commented lines Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Otel metrics working end to end Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Added support for OSD dashboards Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Remove commented code Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Fixed jest test and removed console comments Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * Added test and addressed comments Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * minor Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * fix es lint Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * comments Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * change to get request Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> * remove missed commented line Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com> --------- Signed-off-by: Kavitha Conjeevaram Mohan <mohakavi@amazon.com>
paulstn · Jan 22, 2024 · 971fc8a · 971fc8a
1 parent 9bdd4b1
commit 971fc8a
Show file tree

Hide file tree

Showing 49 changed files with 1,580 additions and 301 deletions.
diff --git a/.cypress/integration/metrics_analytics_test/metrics_analytics.spec.js b/.cypress/integration/metrics_analytics_test/metrics_analytics.spec.js
@@ -55,8 +55,22 @@ describe('Metrics Analytics', () => {
       suppressResizeObserverIssue();
     });
 
+    describe('Check data source picker', () => {
+      it('Index picker should be only available under Otel metric datasource', () => {
+        cy.get('[data-test-subj="metricsDataSourcePicker"]').click();
+        cy.get('[data-test-subj="prometheusOption"]').click();
+        cy.get('[data-test-subj="metricsIndexPicker"]').should('not.exist');
+
+        cy.get('[data-test-subj="metricsDataSourcePicker"]').click();
+        cy.get('[data-test-subj="openTelemetryOption"]').click();
+        cy.get('[data-test-subj="metricsIndexPicker"]').should('exist');
+      });
+    });
+
     describe('Search for metrics in search bar', () => {
       it('Search for metrics in search bar from available metrics', () => {
+        cy.get('[data-test-subj="metricsDataSourcePicker"]').click();
+        cy.get('[data-test-subj="prometheusOption"]').click();
         cy.get('[data-test-subj="metricsSearch"]').type('metric', { wait: 50 });
 
         cy.get('[data-test-subj="metricsListItems_availableMetrics"]')
@@ -76,6 +90,8 @@ describe('Metrics Analytics', () => {
 
     describe('Select and unselect metrics in sidebar', () => {
       it('Select and unselect metrics in sidebar', () => {
+        cy.get('[data-test-subj="metricsDataSourcePicker"]').click();
+        cy.get('[data-test-subj="prometheusOption"]').click();
         cy.get('[data-test-subj="metricsListItems_availableMetrics"]')
           .contains(PPL_METRICS_NAMES[0])
           .trigger('mouseover')
@@ -84,7 +100,7 @@ describe('Metrics Analytics', () => {
           .contains(PPL_METRICS_NAMES[1])
           .trigger('mouseover')
           .click();
-        cy.wait(50);
+        cy.wait(delay/2);
         cy.get('[data-test-subj="metricsListItems_selectedMetrics"]')
           .contains(PPL_METRICS_NAMES[0])
           .should('exist');
@@ -99,7 +115,7 @@ describe('Metrics Analytics', () => {
           .contains(PPL_METRICS_NAMES[1])
           .trigger('mouseover')
           .click();
-        cy.wait(50);
+        cy.wait(delay/2);
         cy.get('[data-test-subj="metricsListItems_availableMetrics"]')
           .contains(PPL_METRICS_NAMES[0])
           .trigger('mouseover')
@@ -113,57 +129,29 @@ describe('Metrics Analytics', () => {
 
     describe('Test Metric Visualizations', () => {
       beforeEach(() => {
+        cy.get('[data-test-subj="metricsDataSourcePicker"]').click();
+        cy.get('[data-test-subj="prometheusOption"]').click();
         cy.get('[data-test-subj="metricsListItems_availableMetrics"]')
           .contains(PPL_METRICS_NAMES[0])
           .trigger('mouseover')
           .click();
       });
 
-      it.only('Resize a Metric visualization in edit mode', () => {
-        cy.get('[data-test-subj="metrics__editView"]')
-          .contains('Edit view')
-          .trigger('mouseover')
-          .click();
-        cy.wait(delay);
-        cy.get('.react-resizable-handle-se')
-          // .eq(1)
-          .trigger('mousedown', { which: 1 })
-          .trigger('mousemove', { clientX: 2000, clientY: 800 })
-          .trigger('mouseup', { force: true });
-        cy.wait(delay);
-        cy.get('[data-test-subj="metrics__saveView"]').trigger('mouseover').click();
-        cy.wait(delay * 3);
-        cy.get('div.react-grid-layout>div').invoke('height').should('match', new RegExp('790'));
-        cy.wait(delay);
-      });
-
       it('Drag and drop a Metric visualization in edit mode', () => {
         cy.get('[data-test-subj="metricsListItems_availableMetrics"]')
           .contains(PPL_METRICS_NAMES[1])
           .trigger('mouseover')
           .click();
-        cy.get('[data-test-subj="metrics__editView"]')
-          .contains('Edit view')
-          .trigger('mouseover')
-          .click();
-        cy.wait(delay);
         cy.get('h5')
           .contains(PPL_METRICS_NAMES[0])
           .trigger('mousedown', { which: 1, force: true })
           .trigger('mousemove', { clientX: 415, clientY: 500 })
           .trigger('mouseup', { force: true });
-        cy.wait(delay);
-        cy.get('[data-test-subj="metrics__saveView"]')
-          .trigger('mouseover')
-          .click({ force: true })
-          .then(() => {
-            cy.wait(delay * 3);
-            cy.get('div.react-grid-layout>div')
-              .eq(1)
-              .invoke('attr', 'style')
-              .should('match', new RegExp('(.*)transform: translate((.*)10px)(.*)'));
-            cy.wait(delay);
-          });
+        cy.wait(delay * 3);
+        cy.get('div.react-grid-layout>div')
+          .eq(1)
+          .invoke('attr', 'style')
+          .should('match', new RegExp('(.*)transform: translate((.*)10px)(.*)'));
       });
 
       it('Change date filter of the Metrics home page', () => {
@@ -175,7 +163,6 @@ describe('Metrics Analytics', () => {
         cy.get('.euiSuperDatePicker__prettyFormat[data-test-subj="superDatePickerShowDatesButton"]')
           .contains('This year')
           .should('exist');
-        cy.wait(delay);
       });
 
       it('Saves metrics to an existing panel', () => {
@@ -228,7 +215,6 @@ const createCustomMetric = ({ testMetricIndex }) => {
     delay: 50,
   });
   cy.get('.euiButton__text').contains('Refresh').trigger('mouseover').click();
-  cy.wait(delay);
   suppressResizeObserverIssue();
   cy.get('button[id="main-content-vis"]').contains('Visualizations').trigger('mouseover').click();
   cy.wait(delay * 2);
@@ -239,11 +225,10 @@ const createCustomMetric = ({ testMetricIndex }) => {
     .focus()
     .type(PPL_METRICS_NAMES[metricIndex], { force: true });
   cy.get('[data-test-subj="eventExplorer__metricSaveName"]').click({ force: true });
-  cy.wait(1000);
+  cy.wait(delay * 10);
   cy.get('[data-test-subj="eventExplorer__querySaveConfirm"]', {
     timeout: COMMAND_TIMEOUT_LONG,
   }).click();
-  cy.wait(delay);
   cy.get('.euiToastHeader__title').contains('successfully').should('exist');
 };
 

diff --git a/.cypress/integration/panels_test/panels.spec.ts b/.cypress/integration/panels_test/panels.spec.ts
@@ -775,7 +775,6 @@ const createSavedObjectPanel = (newName = TEST_PANEL) => {
 };
 
 const addVisualizationsToPanel = (panel, additionalVisualizationIds: string[]) => {
-  console.log('addVisualizationsToPanel', additionalVisualizationIds);
   const additionalVisualizations = additionalVisualizationIds.map((id, idx) => {
     return {
       savedVisualizationId: `observability-visualization:${id}`,
@@ -791,7 +790,6 @@ const addVisualizationsToPanel = (panel, additionalVisualizationIds: string[]) =
     ...panel.attributes.visualizations,
     ...additionalVisualizations,
   ];
-  console.log(panel.attributes);
   cy.request({
     method: 'PUT',
     failOnStatusCode: false,

diff --git a/.cypress/utils/metrics_constants.js b/.cypress/utils/metrics_constants.js
@@ -21,3 +21,9 @@ export const PPL_METRICS = [
 
 export const VIS_TYPE_LINE = 'Time Series';
 export const TESTING_PANEL = 'Mock Testing Panels for Metrics';
+
+export const OTEL_METRICS_NAMES = [
+  '[Otel Metric] ss4o_metrics-otel-dp.duration',
+  '[Otel Metric] ss4o_metrics-otel-dp.http.client.duration',
+  '[Otel Metric] ss4o_metrics-otel-dp.rpc.client.duration',
+];
diff --git a/common/constants/metrics.ts b/common/constants/metrics.ts
@@ -14,6 +14,7 @@ export const PPL_DATASOURCES_REQUEST =
 // redux
 
 export const OBSERVABILITY_CUSTOM_METRIC = 'CUSTOM_METRICS';
+export const OPEN_TELEMETRY = 'OpenTelemetry';
 export const REDUX_SLICE_METRICS = 'metrics';
 
 export const resolutionOptions = [
@@ -33,3 +34,21 @@ export const AGGREGATION_OPTIONS = [
   { value: 'min', text: 'min()' },
   { value: 'max', text: 'max()' },
 ];
+
+export const DATASOURCE_OPTIONS = [
+  {
+    label: 'Prometheus',
+    'data-test-subj': 'prometheusOption',
+  },
+  {
+    label: 'OpenTelemetry',
+    'data-test-subj': 'openTelemetryOption',
+  },
+];
+export const DATA_PREPPER_INDEX_NAME = 'ss4o_metrics-*-*';
+export const METRICS_ANALYTICS_DATA_PREPPER_INDICES_ROUTE =
+  '/api/observability/metrics_analytics/data_prepper_indices';
+
+// Regex pattens
+export const INDEX_DOCUMENT_NAME_PATTERN = /\[Otel Metric\]\s(\S+?-\S+?)\.(\S+)/;
+export const SPAN_RESOLUTION_REGEX = /'(\d+)([smhdwMy])'/;
diff --git a/common/constants/shared.ts b/common/constants/shared.ts
@@ -2,7 +2,6 @@
  * Copyright OpenSearch Contributors
  * SPDX-License-Identifier: Apache-2.0
  */
-import CSS from 'csstype';
 
 // Client route
 export const PPL_BASE = '/api/ppl';
@@ -77,9 +76,11 @@ export const PPL_PATTERNS_DOCUMENTATION_URL =
   'https://github.com/opensearch-project/sql/blob/2.x/docs/user/ppl/cmd/patterns.rst#description';
 export const UI_DATE_FORMAT = 'MM/DD/YYYY hh:mm A';
 export const PPL_DATE_FORMAT = 'YYYY-MM-DD HH:mm:ss.SSSSSS';
+export const OTEL_DATE_FORMAT = 'YYYY-MM-DDTHH:mm:ss';
 export const SPAN_REGEX = /span/;
 
 export const PROMQL_METRIC_SUBTYPE = 'promqlmetric';
+export const OTEL_METRIC_SUBTYPE = 'openTelemetryMetric';
 export const PPL_METRIC_SUBTYPE = 'metric';
 
 export const PPL_SPAN_REGEX = /by\s*span/i;
@@ -131,6 +132,7 @@ export enum VIS_CHART_TYPES {
   Pie = 'pie',
   HeatMap = 'heatmap',
   Text = 'text',
+  Histogram = 'histogram',
 }
 
 export const NUMERICAL_FIELDS = ['short', 'integer', 'long', 'float', 'double'];

diff --git a/common/types/explorer.ts b/common/types/explorer.ts
@@ -6,7 +6,7 @@
 import { History } from 'history';
 import Plotly from 'plotly.js-dist';
 import { QueryManager } from 'common/query_manager';
-import { VIS_CHART_TYPES } from '../../common/constants/shared';
+import { OTEL_METRIC_SUBTYPE, VIS_CHART_TYPES } from '../../common/constants/shared';
 import {
   AGGREGATIONS,
   AVAILABLE_FIELDS,
@@ -174,12 +174,13 @@ export interface SavedVisualization extends SavedObjectAttributes {
   selected_fields: { text: string; tokens: [] };
   selected_timestamp: IField;
   type: string;
-  subType?: 'metric' | 'visualization' | typeof PROMQL_METRIC_SUBTYPE; // exists if sub type is metric
+  subType?: 'metric' | 'visualization'; // exists if sub type is metric
   user_configs?: string;
   units_of_measure?: string;
   application_id?: string;
   dataSources: string; // list of type SelectedDataSources that is stringified
   queryLang: string;
+  metricType?: typeof PROMQL_METRIC_SUBTYPE | typeof OTEL_METRIC_SUBTYPE; // exists if sub type is metric
 }
 
 export interface ExplorerDataType {

diff --git a/common/types/metrics.ts b/common/types/metrics.ts
@@ -4,6 +4,7 @@
  */
 
 import { VisualizationType } from './custom_panels';
+type MetricTypes = 'savedCustomMetric' | 'prometheusMetric' | 'openTelemetryMetric';
 
 export interface MetricType extends VisualizationType {
   id: string;
@@ -13,10 +14,15 @@ export interface MetricType extends VisualizationType {
   w: number;
   h: number;
   query: {
-    type: 'savedCustomMetric' | 'prometheusMetric';
+    type: MetricTypes;
     aggregation: string;
     attributesGroupBy: string[];
     catalog: string;
     availableAttributes?: string[];
   };
 }
+
+export interface OptionType {
+  label: string;
+  'data-test-subj': string;
+}
diff --git a/public/components/application_analytics/helpers/utils.tsx b/public/components/application_analytics/helpers/utils.tsx
@@ -216,7 +216,7 @@ export const calculateAvailability = async (
   for (let i = 0; i < savedVisualizationsIds.length; i++) {
     const visualizationId = savedVisualizationsIds[i];
     // Fetches data for visualization
-    const visData = await fetchVisualizationById(http, visualizationId, (value: string) =>
+    const visData = await fetchVisualizationById(visualizationId, (value: string) =>
       console.error(value)
     );
 

diff --git a/public/components/common/query_utils/__tests__/query_utils.test.tsx b/public/components/common/query_utils/__tests__/query_utils.test.tsx
@@ -108,6 +108,7 @@ describe('Query Utils', () => {
       span: '1',
       resolution: 'h',
     };
+
     describe('updateCatalogVisualizationQuery', () => {
       it('should build plain promQL series query', () => {
         const query = updateCatalogVisualizationQuery(defaultQueryMetaData);
@@ -125,12 +126,14 @@ describe('Query Utils', () => {
       it('should set timestamps and default resolution', () => {
         const [startDate, endDate] = ['2023-11-11', '2023-12-11'];
         const [start, end] = [1699660800, 1702252800]; // 2023-11-11 to 2023-12-11
-        const query = preprocessMetricQuery({
-          metaData: { queryMetaData: defaultQueryMetaData },
+        const currentQuery =
+          "source = test_catalog.query_range('count by(one,two) (metric)', 1699660800, 1702252800, '1d')";
+        const expectedQuery = preprocessMetricQuery({
+          metaData: { query: currentQuery, queryMetaData: defaultQueryMetaData },
           startTime: startDate,
           endTime: endDate,
         });
-        expect(query).toMatch(new RegExp(`, ${start}, ${end}, '1d'`));
+        expect(expectedQuery).toMatch(new RegExp(`, ${start}, ${end}, '1d'`));
       });
     });
   });