Skip to content

Commit c043adf

Browse files
IanHoangIan Hoang
and
Ian Hoang
authored
Revert publisher back to results_publisher (opensearch-project#353)
Signed-off-by: Ian Hoang <hoangia@amazon.com> Co-authored-by: Ian Hoang <hoangia@amazon.com>
1 parent e04fa7f commit c043adf

19 files changed

+131
-130
lines changed

README.md

+2-2
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ After the test execution, a summary report is written to the command line:
110110

111111
Creating Your Own Workloads
112112
---------------------------
113-
For more information on how users can create their own workloads, see [the Create Workload Guide](https://github.com/opensearch-project/opensearch-benchmark/blob/main/CREATE_WORKLOAD_GUIDE.md)
113+
For more information on how users can create their own workloads, see [the Create Workload Guide](./CREATE_WORKLOAD_GUIDE.md)
114114

115115
Getting help
116116
------------
@@ -142,4 +142,4 @@ Unless required by applicable law or agreed to in writing, software
142142
distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
143143
WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
144144
License for the specific language governing permissions and limitations under
145-
the License.
145+
the License.

it/resources/benchmark-in-memory-it.ini

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ opensearch.src.subdir = opensearch
1515
[benchmarks]
1616
local.dataset.cache = ${CONFIG_DIR}/benchmarks/data
1717

18-
[reporting]
18+
[results_publishing]
1919
datastore.type = in-memory
2020

2121

it/resources/benchmark-os-it.ini

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ opensearch.src.subdir = opensearch
1515
[benchmarks]
1616
local.dataset.cache = ${CONFIG_DIR}/benchmarks/data
1717

18-
[reporting]
18+
[results_publishing]
1919
datastore.type = opensearch
2020
datastore.host = localhost
2121
datastore.port = 10200

osbenchmark/benchmark.py

+9-9
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@
3535

3636
from osbenchmark import PROGRAM_NAME, BANNER, FORUM_LINK, SKULL, check_python_version, doc_link, telemetry
3737
from osbenchmark import version, actor, config, paths, \
38-
test_execution_orchestrator, publisher, \
38+
test_execution_orchestrator, results_publisher, \
3939
metrics, workload, chart_generator, exceptions, log
4040
from osbenchmark.builder import provision_config, builder
4141
from osbenchmark.workload_generator import workload_generator
@@ -815,11 +815,11 @@ def configure_connection_params(arg_parser, args, cfg):
815815
arg_parser.error("--target-hosts and --client-options must define the same keys for multi cluster setups.")
816816

817817

818-
def configure_reporting_params(args, cfg):
819-
cfg.add(config.Scope.applicationOverride, "reporting", "format", args.results_format)
820-
cfg.add(config.Scope.applicationOverride, "reporting", "values", args.show_in_results)
821-
cfg.add(config.Scope.applicationOverride, "reporting", "output.path", args.results_file)
822-
cfg.add(config.Scope.applicationOverride, "reporting", "numbers.align", args.results_numbers_align)
818+
def configure_results_publishing_params(args, cfg):
819+
cfg.add(config.Scope.applicationOverride, "results_publishing", "format", args.results_format)
820+
cfg.add(config.Scope.applicationOverride, "results_publishing", "values", args.show_in_results)
821+
cfg.add(config.Scope.applicationOverride, "results_publishing", "output.path", args.results_file)
822+
cfg.add(config.Scope.applicationOverride, "results_publishing", "numbers.align", args.results_numbers_align)
823823

824824

825825
def dispatch_sub_command(arg_parser, args, cfg):
@@ -830,8 +830,8 @@ def dispatch_sub_command(arg_parser, args, cfg):
830830

831831
try:
832832
if sub_command == "compare":
833-
configure_reporting_params(args, cfg)
834-
publisher.compare(cfg, args.baseline, args.contender)
833+
configure_results_publishing_params(args, cfg)
834+
results_publisher.compare(cfg, args.baseline, args.contender)
835835
elif sub_command == "list":
836836
cfg.add(config.Scope.applicationOverride, "system", "list.config.option", args.configuration)
837837
cfg.add(config.Scope.applicationOverride, "system", "list.test_executions.max_results", args.limit)
@@ -903,7 +903,7 @@ def dispatch_sub_command(arg_parser, args, cfg):
903903
cfg.add(config.Scope.applicationOverride, "builder", "preserve.install", convert.to_bool(args.preserve_install))
904904
cfg.add(config.Scope.applicationOverride, "builder", "skip.rest.api.check", convert.to_bool(args.skip_rest_api_check))
905905

906-
configure_reporting_params(args, cfg)
906+
configure_results_publishing_params(args, cfg)
907907

908908
execute_test(cfg, args.kill_running_processes)
909909
elif sub_command == "generate":

osbenchmark/config.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -114,7 +114,7 @@ def auto_load_local_config(base_config, additional_sections=None, config_file_cl
114114
cfg.load_config(auto_upgrade=True)
115115
# we override our some configuration with the one from the coordinator because it may contain more entries and we should be
116116
# consistent across all nodes here.
117-
cfg.add_all(base_config, "reporting")
117+
cfg.add_all(base_config, "results_publishing")
118118
cfg.add_all(base_config, "workloads")
119119
cfg.add_all(base_config, "provision_configs")
120120
cfg.add_all(base_config, "distributions")

osbenchmark/metrics.py

+21-20
Original file line numberDiff line numberDiff line change
@@ -179,26 +179,26 @@ class OsClientFactory:
179179

180180
def __init__(self, cfg):
181181
self._config = cfg
182-
host = self._config.opts("reporting", "datastore.host")
183-
port = self._config.opts("reporting", "datastore.port")
184-
secure = convert.to_bool(self._config.opts("reporting", "datastore.secure"))
185-
user = self._config.opts("reporting", "datastore.user")
182+
host = self._config.opts("results_publishing", "datastore.host")
183+
port = self._config.opts("results_publishing", "datastore.port")
184+
secure = convert.to_bool(self._config.opts("results_publishing", "datastore.secure"))
185+
user = self._config.opts("results_publishing", "datastore.user")
186186

187-
metrics_amazon_aws_log_in = self._config.opts("reporting", "datastore.amazon_aws_log_in",
187+
metrics_amazon_aws_log_in = self._config.opts("results_publishing", "datastore.amazon_aws_log_in",
188188
default_value=None, mandatory=False)
189189
metrics_aws_access_key_id = None
190190
metrics_aws_secret_access_key = None
191191
metrics_aws_region = None
192192
metrics_aws_service = None
193193

194194
if metrics_amazon_aws_log_in == 'config':
195-
metrics_aws_access_key_id = self._config.opts("reporting", "datastore.aws_access_key_id",
195+
metrics_aws_access_key_id = self._config.opts("results_publishing", "datastore.aws_access_key_id",
196196
default_value=None, mandatory=False)
197-
metrics_aws_secret_access_key = self._config.opts("reporting", "datastore.aws_secret_access_key",
197+
metrics_aws_secret_access_key = self._config.opts("results_publishing", "datastore.aws_secret_access_key",
198198
default_value=None, mandatory=False)
199-
metrics_aws_region = self._config.opts("reporting", "datastore.region",
199+
metrics_aws_region = self._config.opts("results_publishing", "datastore.region",
200200
default_value=None, mandatory=False)
201-
metrics_aws_service = self._config.opts("reporting", "datastore.service",
201+
metrics_aws_service = self._config.opts("results_publishing", "datastore.service",
202202
default_value=None, mandatory=False)
203203
elif metrics_amazon_aws_log_in == 'environment':
204204
metrics_aws_access_key_id = os.getenv("OSB_DATASTORE_AWS_ACCESS_KEY_ID", default=None)
@@ -233,14 +233,14 @@ def __init__(self, cfg):
233233
password = os.environ["OSB_DATASTORE_PASSWORD"]
234234
except KeyError:
235235
try:
236-
password = self._config.opts("reporting", "datastore.password")
236+
password = self._config.opts("results_publishing", "datastore.password")
237237
except exceptions.ConfigError:
238238
raise exceptions.ConfigError(
239-
"No password configured through [reporting] configuration or OSB_DATASTORE_PASSWORD environment variable."
239+
"No password configured through [results_publishing] configuration or OSB_DATASTORE_PASSWORD environment variable."
240240
) from None
241-
verify = self._config.opts("reporting", "datastore.ssl.verification_mode", default_value="full", mandatory=False) != "none"
242-
ca_path = self._config.opts("reporting", "datastore.ssl.certificate_authorities", default_value=None, mandatory=False)
243-
self.probe_version = self._config.opts("reporting", "datastore.probe.cluster_version", default_value=True, mandatory=False)
241+
verify = self._config.opts("results_publishing", "datastore.ssl.verification_mode", default_value="full", mandatory=False) != "none"
242+
ca_path = self._config.opts("results_publishing", "datastore.ssl.certificate_authorities", default_value=None, mandatory=False)
243+
self.probe_version = self._config.opts("results_publishing", "datastore.probe.cluster_version", default_value=True, mandatory=False)
244244

245245
# Instead of duplicating code, we're just adapting the metrics store specific properties to match the regular client options.
246246
client_options = {
@@ -280,8 +280,9 @@ class IndexTemplateProvider:
280280
def __init__(self, cfg):
281281
self._config = cfg
282282
self.script_dir = self._config.opts("node", "benchmark.root")
283-
self._number_of_shards = self._config.opts("reporting", "datastore.number_of_shards", default_value=None, mandatory=False)
284-
self._number_of_replicas = self._config.opts("reporting", "datastore.number_of_replicas", default_value=None, mandatory=False)
283+
self._number_of_shards = self._config.opts("results_publishing", "datastore.number_of_shards", default_value=None, mandatory=False)
284+
self._number_of_replicas = self._config.opts("results_publishing", "datastore.number_of_replicas",
285+
default_value=None, mandatory=False)
285286

286287
def metrics_template(self):
287288
return self._read("metrics-template")
@@ -357,7 +358,7 @@ def metrics_store(cfg, read_only=True, workload=None, test_procedure=None, provi
357358

358359

359360
def metrics_store_class(cfg):
360-
if cfg.opts("reporting", "datastore.type") == "opensearch":
361+
if cfg.opts("results_publishing", "datastore.type") == "opensearch":
361362
return OsMetricsStore
362363
else:
363364
return InMemoryMetricsStore
@@ -1207,7 +1208,7 @@ def test_execution_store(cfg):
12071208
:return: A test_execution store implementation.
12081209
"""
12091210
logger = logging.getLogger(__name__)
1210-
if cfg.opts("reporting", "datastore.type") == "opensearch":
1211+
if cfg.opts("results_publishing", "datastore.type") == "opensearch":
12111212
logger.info("Creating OS test execution store")
12121213
return CompositeTestExecutionStore(EsTestExecutionStore(cfg), FileTestExecutionStore(cfg))
12131214
else:
@@ -1222,7 +1223,7 @@ def results_store(cfg):
12221223
:return: A test_execution store implementation.
12231224
"""
12241225
logger = logging.getLogger(__name__)
1225-
if cfg.opts("reporting", "datastore.type") == "opensearch":
1226+
if cfg.opts("results_publishing", "datastore.type") == "opensearch":
12261227
logger.info("Creating OS results store")
12271228
return OsResultsStore(cfg)
12281229
else:
@@ -1690,7 +1691,7 @@ def __call__(self):
16901691
op_type = task.operation.type
16911692
error_rate = self.error_rate(t, op_type)
16921693
duration = self.duration(t)
1693-
if task.operation.include_in_reporting or error_rate > 0:
1694+
if task.operation.include_in_results_publishing or error_rate > 0:
16941695
self.logger.debug("Gathering request metrics for [%s].", t)
16951696
result.add_op_metrics(
16961697
t,

osbenchmark/resources/benchmark.ini

+1-1
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,7 @@ opensearch.src.subdir = opensearch
1515
[benchmarks]
1616
local.dataset.cache = ${CONFIG_DIR}/benchmarks/data
1717

18-
[reporting]
18+
[results_publishing]
1919
datastore.type = in-memory
2020
datastore.host =
2121
datastore.port =

osbenchmark/publisher.py osbenchmark/results_publisher.py

+11-11
Original file line numberDiff line numberDiff line change
@@ -99,14 +99,14 @@ def format_as_csv(headers, data):
9999
class SummaryResultsPublisher:
100100
def __init__(self, results, config):
101101
self.results = results
102-
self.results_file = config.opts("reporting", "output.path")
103-
self.results_format = config.opts("reporting", "format")
104-
self.numbers_align = config.opts("reporting", "numbers.align",
102+
self.results_file = config.opts("results_publishing", "output.path")
103+
self.results_format = config.opts("results_publishing", "format")
104+
self.numbers_align = config.opts("results_publishing", "numbers.align",
105105
mandatory=False, default_value="right")
106-
reporting_values = config.opts("reporting", "values")
107-
self.publish_all_values = reporting_values == "all"
108-
self.publish_all_percentile_values = reporting_values == "all-percentiles"
109-
self.show_processing_time = convert.to_bool(config.opts("reporting", "output.processingtime",
106+
results_publishing_values = config.opts("results_publishing", "values")
107+
self.publish_all_values = results_publishing_values == "all"
108+
self.publish_all_percentile_values = results_publishing_values == "all-percentiles"
109+
self.show_processing_time = convert.to_bool(config.opts("results_publishing", "output.processingtime",
110110
mandatory=False, default_value=False))
111111
self.cwd = config.opts("node", "benchmark.cwd")
112112

@@ -317,12 +317,12 @@ def _line(self, k, task, v, unit, converter=lambda x: x, force=False):
317317

318318
class ComparisonResultsPublisher:
319319
def __init__(self, config):
320-
self.results_file = config.opts("reporting", "output.path")
321-
self.results_format = config.opts("reporting", "format")
322-
self.numbers_align = config.opts("reporting", "numbers.align",
320+
self.results_file = config.opts("results_publishing", "output.path")
321+
self.results_format = config.opts("results_publishing", "format")
322+
self.numbers_align = config.opts("results_publishing", "numbers.align",
323323
mandatory=False, default_value="right")
324324
self.cwd = config.opts("node", "benchmark.cwd")
325-
self.show_processing_time = convert.to_bool(config.opts("reporting", "output.processingtime",
325+
self.show_processing_time = convert.to_bool(config.opts("results_publishing", "output.processingtime",
326326
mandatory=False, default_value=False))
327327
self.plain = False
328328

osbenchmark/test_execution_orchestrator.py

+2-2
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232

3333
from osbenchmark import actor, config, doc_link, \
3434
worker_coordinator, exceptions, builder, metrics, \
35-
publisher, workload, version, PROGRAM_NAME
35+
results_publisher, workload, version, PROGRAM_NAME
3636
from osbenchmark.utils import console, opts, versions
3737

3838

@@ -250,7 +250,7 @@ def on_benchmark_complete(self, new_metrics):
250250
self.test_execution.add_results(final_results)
251251
self.test_execution_store.store_test_execution(self.test_execution)
252252
metrics.results_store(self.cfg).store_results(self.test_execution)
253-
publisher.summarize(final_results, self.cfg)
253+
results_publisher.summarize(final_results, self.cfg)
254254
else:
255255
self.logger.info("Suppressing output of summary results. Cancelled = [%r], Error = [%r].", self.cancelled, self.error)
256256
self.metrics_store.close()

osbenchmark/worker_coordinator/worker_coordinator.py

+6-6
Original file line numberDiff line numberDiff line change
@@ -533,7 +533,7 @@ def __init__(self, target, config, os_client_factory_class=client.OsClientFactor
533533
# which client ids are assigned to which workers?
534534
self.clients_per_worker = {}
535535

536-
self.progress_publisher = console.progress()
536+
self.progress_results_publisher = console.progress()
537537
self.progress_counter = 0
538538
self.quiet = False
539539
self.allocations = None
@@ -608,7 +608,7 @@ def prepare_benchmark(self, t):
608608
self.test_procedure = select_test_procedure(self.config, self.workload)
609609
self.quiet = self.config.opts("system", "quiet.mode", mandatory=False, default_value=False)
610610
downsample_factor = int(self.config.opts(
611-
"reporting", "metrics.request.downsample.factor",
611+
"results_publishing", "metrics.request.downsample.factor",
612612
mandatory=False, default_value=1))
613613
self.metrics_store = metrics.metrics_store(cfg=self.config,
614614
workload=self.workload.name,
@@ -792,7 +792,7 @@ def finished(self):
792792
return self.current_step == self.number_of_steps
793793

794794
def close(self):
795-
self.progress_publisher.finish()
795+
self.progress_results_publisher.finish()
796796
if self.metrics_store and self.metrics_store.opened:
797797
self.metrics_store.close()
798798

@@ -818,9 +818,9 @@ def update_progress_message(self, task_finished=False):
818818

819819
num_clients = max(len(progress_per_client), 1)
820820
total_progress = sum(progress_per_client) / num_clients
821-
self.progress_publisher.print("Running %s" % tasks, "[%3d%% done]" % (round(total_progress * 100)))
821+
self.progress_results_publisher.print("Running %s" % tasks, "[%3d%% done]" % (round(total_progress * 100)))
822822
if task_finished:
823-
self.progress_publisher.finish()
823+
self.progress_results_publisher.finish()
824824

825825
def post_process_samples(self):
826826
# we do *not* do this here to avoid concurrent updates (actors are single-threaded) but rather to make it clear that we use
@@ -1032,7 +1032,7 @@ def receiveMsg_StartWorker(self, msg, sender):
10321032
self.worker_id = msg.worker_id
10331033
self.config = load_local_config(msg.config)
10341034
self.on_error = self.config.opts("worker_coordinator", "on.error")
1035-
self.sample_queue_size = int(self.config.opts("reporting", "sample.queue.size", mandatory=False, default_value=1 << 20))
1035+
self.sample_queue_size = int(self.config.opts("results_publishing", "sample.queue.size", mandatory=False, default_value=1 << 20))
10361036
self.workload = msg.workload
10371037
workload.set_absolute_data_path(self.config, self.workload)
10381038
self.client_allocations = msg.client_allocations

osbenchmark/workload/loader.py

+3-3
Original file line numberDiff line numberDiff line change
@@ -1365,7 +1365,7 @@ def _create_test_procedures(self, workload_spec):
13651365
task = self.parse_task(op, ops, name)
13661366
schedule.append(task)
13671367

1368-
# verify we don't have any duplicate task names (which can be confusing / misleading in reporting).
1368+
# verify we don't have any duplicate task names (which can be confusing / misleading in results_publishing).
13691369
known_task_names = set()
13701370
for task in schedule:
13711371
for sub_task in task:
@@ -1523,8 +1523,8 @@ def parse_operation(self, op_spec, error_ctx="operations"):
15231523

15241524
try:
15251525
op = workload.OperationType.from_hyphenated_string(op_type_name)
1526-
if "include-in-reporting" not in params:
1527-
params["include-in-reporting"] = not op.admin_op
1526+
if "include-in-results_publishing" not in params:
1527+
params["include-in-results_publishing"] = not op.admin_op
15281528
self.logger.debug("Using built-in operation type [%s] for operation [%s].", op_type_name, op_name)
15291529
except KeyError:
15301530
self.logger.info("Using user-provided operation type [%s] for operation [%s].", op_type_name, op_name)

0 commit comments

Comments
 (0)