Sample Data Tooling Refactor (#70)

* Refactored file structure and modified sys.path

Signed-off-by: Huy Nguyen <wronghuy@amazon.com>

* Added another directory level for Dockerfile

Signed-off-by: Huy Nguyen <wronghuy@amazon.com>

* Removed a directory structure layer

Signed-off-by: Huy Nguyen <wronghuy@amazon.com>

* Added license headers to unit tests

Signed-off-by: Huy Nguyen <wronghuy@amazon.com>

* Removed unnecessary comments from Dockerfile

Signed-off-by: Huy Nguyen <wronghuy@amazon.com>

Signed-off-by: Huy Nguyen <wronghuy@amazon.com>

Author: huyaboo

src/playground/sample_data_tooling/Dockerfile

@@ -1,9 +1,15 @@
-# syntax=docker/dockerfile:1
+# SPDX-License-Identifier: Apache-2.0
+#
+# The OpenSearch Contributors require contributions made to
+# this file be licensed under the Apache-2.0 license or a
+# compatible open source license.
 
 FROM python:3
 
 COPY requirements.txt requirements.txt
 
 RUN pip install -r requirements.txt
 
-COPY . .
+RUN mkdir sample_data_tooling
+
+COPY . sample_data_tooling

src/playground/sample_data_tooling/README.md

@@ -60,7 +60,7 @@ In order to deploy this tooling to Playground, first ensure that [`kubectl`](htt
 Once `kubectl` is installed and the Kubernetes cluster is running, first configure secrets so the container can pull credentials from the cluster.
 
 ```
-kubectl create secret generic sample-data-secret --from-literal='username=<admin username>' --from-literal='password=<admin password>' --from-literal='host=opensearch-cluster-leader'
+kubectl create secret generic sample-data-secret --from-literal='SAMPLE_DATA_USERNAME=<username>' --from-literal='SAMPLE_DATA_PASSWORD=<password>' --from-literal='SAMPLE_DATA_HOST=opensearch-cluster-leader'
 ```
 
 Then, from this directory, change directory into the job and cronjob deployment files and deploy them.

src/playground/sample_data_tooling/sample_data_authentication/sample_data_basic_authentication.py

@@ -8,8 +8,8 @@
 import sys
 
 # Adds parent directory "/sample_data_tooling" to sys.path
-sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
-from sample_data_authentication.sample_data_authentication import Authentication
+sys.path.append(path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_authentication.sample_data_authentication import Authentication
 
 
 class BasicAuthentication(Authentication):

src/playground/sample_data_tooling/sample_data_indices/sample_data_indices.py

@@ -10,8 +10,8 @@
 import sys
 
 # Adds parent directory "/sample_data_tooling" to sys.path
-sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
-from sample_data_ingestor.sample_data_ingestor import ingest
+sys.path.append(path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_ingestor.sample_data_ingestor import ingest
 
 
 class SampleDataIndex:

src/playground/sample_data_tooling/sample_data_plugins/ad_plugin_class.py

@@ -11,9 +11,9 @@
 import sys
 
 # Adds parent directory "/sample_data_tooling" to sys.path
-sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
-from sample_data_authentication.sample_data_authentication import Authentication
-from sample_data_plugins.sample_data_plugin import Plugin
+sys.path.append(path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_authentication.sample_data_authentication import Authentication
+from sample_data_tooling.sample_data_plugins.sample_data_plugin import Plugin
 
 
 class AnomalyDetection(Plugin):

src/playground/sample_data_tooling/sample_data_plugins/ad_plugin_data_config/data_trend_functions.py src/playground/sample_data_tooling/sample_data_plugins/ad_plugin_data_config/average_trend_class.py

@@ -10,9 +10,9 @@
 import sys
 
 # Adds parent directory "/sample_data_tooling" to sys.path
-sys.path.append(path.dirname(path.dirname(path.abspath(__file__))))
-from sample_data_authentication.sample_data_authentication import Authentication
-from sample_data_commons.utils import validate_filename
+sys.path.append(path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_authentication.sample_data_authentication import Authentication
+from sample_data_tooling.sample_data_commons.utils import validate_filename
 
 
 class Plugin:

src/playground/sample_data_tooling/sample_data_plugins/sample_data_plugin.py

@@ -8,8 +8,8 @@
 import os
 
 # Adds parent directory sample_data_tooling to sys.path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-from sample_data_authentication.sample_data_basic_authentication import BasicAuthentication
+sys.path.append(os.path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_authentication.sample_data_basic_authentication import BasicAuthentication
 
 
 # Test that basic authentication works and returns as expected

src/playground/sample_data_tooling/sample_data_tests/sample_data_authentication_tests/unit-test/sample_data_authentication_unit_test.py

@@ -1,3 +1,8 @@
+"""
+Copyright OpenSearch Contributors
+SPDX-License-Identifier: Apache-2.0
+"""
+
 # Test functions
 import os
 import sys
@@ -7,8 +12,8 @@
 from types import NoneType
 
 # Adds parent directory sample_data_tooling to sys.path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-import sample_data_generator.sample_data_generator as generator
+sys.path.append(os.path.abspath(__file__).split("sample_data_tooling")[0])
+import sample_data_tooling.sample_data_generator.sample_data_generator as generator
 
 
 # This path is for getting the parent directory sample_data_generator_tests
@@ -78,7 +83,7 @@ def test_valid_choose_field():
         assert type(generator.choose_field("keyword", ["email"])) is str
         assert "@" in generator.choose_field("keyword", ["email"])
         assert type(generator.choose_field("date detection string")) is bool
-        assert type(generator.choose_field("numeric detection string")) is bool 
+        assert type(generator.choose_field("numeric detection string")) is bool
         # One of Faker's standard providers
         ssn = generator.choose_field("ssn")
         assert type(ssn) is str and "-" == ssn[3] and "-" == ssn[6]
@@ -158,7 +163,7 @@ def test_valid_generate_data():
         test_2 = json.loads(generator.generate_data(valid_json_mapping))
         assert len(test_2) == 3
         for field in test_2:
-                assert test_2[field] != None        
+                assert test_2[field] != None
 
         test_3 = json.loads(generator.generate_data(valid_json_string_shorthand, False))
         assert len(test_3) == 3

src/playground/sample_data_tooling/sample_data_tests/sample_data_generator_tests/unit-test/sample_data_generator_unit_test.py

@@ -10,8 +10,8 @@
 import os
 
 # Adds the directory "/sample_data_tooling" to sys.path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-from sample_data_indices.sample_data_indices import SampleDataIndex
+sys.path.append(os.path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_indices.sample_data_indices import SampleDataIndex
 
 
 # Constants

src/playground/sample_data_tooling/sample_data_tests/sample_data_indices_tests/unit-test/sample_data_indices_unit_test.py

@@ -8,9 +8,9 @@
 import os
 
 # Adds the directory "/sample_data_tooling" to sys.path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-from sample_data_plugins.ad_plugin_class import AnomalyDetection
-from sample_data_authentication.sample_data_basic_authentication import BasicAuthentication
+sys.path.append(os.path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_plugins.ad_plugin_class import AnomalyDetection
+from sample_data_tooling.sample_data_authentication.sample_data_basic_authentication import BasicAuthentication
 
 
 # Constants

src/playground/sample_data_tooling/sample_data_tests/sample_data_plugin_tests/unit-test/ad_plugin_class_unit_test.py

@@ -8,9 +8,9 @@
 import os
 
 # Adds the directory "/sample_data_tooling" to sys.path
-sys.path.append(os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))))
-from sample_data_plugins.sample_data_plugin import Plugin
-from sample_data_authentication.sample_data_basic_authentication import BasicAuthentication
+sys.path.append(os.path.abspath(__file__).split("sample_data_tooling")[0])
+from sample_data_tooling.sample_data_plugins.sample_data_plugin import Plugin
+from sample_data_tooling.sample_data_authentication.sample_data_basic_authentication import BasicAuthentication
 
 
 # Constants