Skip to content

Commit 229c302

Browse files
ndimidukndimiduk
committed
HBASE-27832 Introduce hbase overlay
1 parent 2bb2fdb commit 229c302

16 files changed

+1157
-0
lines changed

‎hbase-kubernetes-deployment/components/hbase/hadoop-metrics2-hbase.properties

+46
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,46 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# syntax: [prefix].[source|sink].[instance].[options]
18+
# See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
19+
20+
*.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink
21+
# default sampling period
22+
*.period=10
23+
24+
# Below are some examples of sinks that could be used
25+
# to monitor different hbase daemons.
26+
27+
# hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink
28+
# hbase.sink.file-all.filename=all.metrics
29+
30+
# hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink
31+
# hbase.sink.file0.context=hmaster
32+
# hbase.sink.file0.filename=master.metrics
33+
34+
# hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink
35+
# hbase.sink.file1.context=thrift-one
36+
# hbase.sink.file1.filename=thrift-one.metrics
37+
38+
# hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink
39+
# hbase.sink.file2.context=thrift-two
40+
# hbase.sink.file2.filename=thrift-one.metrics
41+
42+
# hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink
43+
# hbase.sink.file3.context=rest
44+
# hbase.sink.file3.filename=rest.metrics
45+
46+
# Read from src rather than copy over like this.

‎hbase-kubernetes-deployment/components/hbase/hbase-site.xml

+75
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<!--
3+
Licensed to the Apache Software Foundation (ASF) under one
4+
or more contributor license agreements. See the NOTICE file
5+
distributed with this work for additional information
6+
regarding copyright ownership. The ASF licenses this file
7+
to you under the Apache License, Version 2.0 (the
8+
"License"); you may not use this file except in compliance
9+
with the License. You may obtain a copy of the License at
10+
11+
http://www.apache.org/licenses/LICENSE-2.0
12+
13+
Unless required by applicable law or agreed to in writing, software
14+
distributed under the License is distributed on an "AS IS" BASIS,
15+
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
16+
See the License for the specific language governing permissions and
17+
limitations under the License.
18+
-->
19+
<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
20+
<configuration>
21+
<property>
22+
<name>hbase.cluster.distributed</name>
23+
<value>true</value>
24+
</property>
25+
<property>
26+
<name>hbase.rootdir</name>
27+
<value>hdfs://${env.HADOOP_SERVICE}:8020/hbase</value>
28+
</property>
29+
<property>
30+
<name>hbase.zookeeper.quorum</name>
31+
<value>${env.HBASE_ZOOKEEPER_QUORUM}</value>
32+
</property>
33+
<property>
34+
<!--In k8s context, master notion of name doesn't always agree; use RS name.
35+
Seems to fix case where RS was showing up w/ IP instead of name because
36+
Master told it use its understanding of what its name was. See
37+
HBASE-18226 and then HBASE-24667-->
38+
<name>hbase.unsafe.regionserver.hostname.disable.master.reversedns</name>
39+
<value>true</value>
40+
</property>
41+
<property>
42+
<name>dfs.client.use.datanode.hostname</name>
43+
<value>true</value>
44+
</property>
45+
<property>
46+
<name>hbase.ssl.enabled</name>
47+
<!-- Needs upstream fix. See https://github.com/eclipse/jetty.project/issues/4385-->
48+
<value>${env.HBASE_SSL_ENABLED}</value>
49+
</property>
50+
<property>
51+
<name>hbase.master.cleaner.interval</name>
52+
<!-- Needs to clean up oldWAL and archived hfiles every 10 seconds -->
53+
<value>10000</value>
54+
</property>
55+
<property>
56+
<name>hbase.regionserver.handler.count</name>
57+
<value>45</value>
58+
</property>
59+
<property>
60+
<name>hbase.netty.eventloop.rpcserver.thread.count</name>
61+
<value>1</value>
62+
<description>See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default
63+
is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least.</description>
64+
</property>
65+
<property>
66+
<name>hbase.netty.worker.count</name>
67+
<value>1</value>
68+
<description>See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default
69+
is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least.</description>
70+
</property>
71+
<property>
72+
<name>hbase.regionserver.throughput.controller</name>
73+
<value>org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController</value>
74+
</property>
75+
</configuration>

‎hbase-kubernetes-deployment/components/hbase/kustomization.yaml

+40
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
---
17+
apiVersion: kustomize.config.k8s.io/v1alpha1
18+
kind: Component
19+
20+
commonLabels:
21+
app: hadoop
22+
23+
configMapGenerator:
24+
- name: hbase-configuration
25+
files:
26+
- hbase-site.xml
27+
- hadoop-metrics2-hbase.properties
28+
- log4j2.properties=log4j2.properties.hbase
29+
- name: hbck2-configuration
30+
# Add an hbck2-configuration. Uses log4j1 vs log4j2.
31+
files:
32+
- hbase-site.xml
33+
- hadoop-metrics2-hbase.properties
34+
- log4j.properties=log4j.properties.hbase
35+
36+
resources:
37+
- m-service.yaml
38+
- m-statefulset.yaml
39+
- rs-service.yaml
40+
- rs-statefulset.yaml

‎hbase-kubernetes-deployment/components/hbase/log4j.properties.hbase

+68
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
# Licensed to the Apache Software Foundation (ASF) under one
2+
# or more contributor license agreements. See the NOTICE file
3+
# distributed with this work for additional information
4+
# regarding copyright ownership. The ASF licenses this file
5+
# to you under the Apache License, Version 2.0 (the
6+
# "License"); you may not use this file except in compliance
7+
# with the License. You may obtain a copy of the License at
8+
#
9+
# http://www.apache.org/licenses/LICENSE-2.0
10+
#
11+
# Unless required by applicable law or agreed to in writing, software
12+
# distributed under the License is distributed on an "AS IS" BASIS,
13+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
# See the License for the specific language governing permissions and
15+
# limitations under the License.
16+
17+
# Define some default values that can be overridden by system properties
18+
hbase.root.logger=INFO,console
19+
hbase.log.dir=.
20+
hbase.log.file=hbase.log
21+
22+
# Define the root logger to the system property "hbase.root.logger".
23+
log4j.rootLogger=${hbase.root.logger}
24+
25+
# Logging Threshold
26+
log4j.threshold=ALL
27+
28+
#
29+
# Daily Rolling File Appender
30+
#
31+
log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
32+
log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
33+
34+
# Rollver at midnight
35+
log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
36+
37+
# 30-day backup
38+
#log4j.appender.DRFA.MaxBackupIndex=30
39+
log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
40+
# Debugging Pattern format
41+
log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n
42+
43+
44+
#
45+
# console
46+
# Add "console" to rootlogger above if you want to use this
47+
#
48+
log4j.appender.console=org.apache.log4j.ConsoleAppender
49+
log4j.appender.console.target=System.err
50+
log4j.appender.console.layout=org.apache.log4j.PatternLayout
51+
log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n
52+
53+
# Custom Logging levels
54+
55+
#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
56+
57+
log4j.logger.org.apache.hadoop=WARN
58+
log4j.logger.org.apache.zookeeper=ERROR
59+
log4j.logger.org.apache.hadoop.hbase=DEBUG
60+
61+
#These settings are workarounds against spurious logs from the minicluster.
62+
#See HBASE-4709
63+
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN
64+
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN
65+
log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN
66+
log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN
67+
# Enable this to get detailed connection error/retry logging.
68+
# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE

0 commit comments

Comments
 (0)