HBASE-27832 Introduce hbase overlay

.gitignore

@@ -21,3 +21,6 @@ linklint-*.zip
 linklint/
 .checkstyle
 **/.checkstyle
+
+# detritus produced by kuttl
+kubeconfig*

dev-support/jenkins/Dockerfile

@@ -20,16 +20,29 @@
 FROM hadolint/hadolint:latest-debian as hadolint
 
 FROM maven:3.8-jdk-8
-
-# hadolint ignore=DL3008
-RUN apt-get -q update && apt-get -q install --no-install-recommends -y \
-       binutils \
-       git \
-       rsync \
-       shellcheck \
-       wget && \
-    apt-get clean && \
-    rm -rf /var/lib/apt/lists/*
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
+# hadolint ignore=SC1091
+RUN install -m 0755 -d /etc/apt/keyrings \
+    && curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg \
+    && chmod a+r /etc/apt/keyrings/docker.gpg \
+    && echo \
+        "deb [arch="$(dpkg --print-architecture)" signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian \
+        "$(. /etc/os-release && echo "$VERSION_CODENAME")" stable" | \
+        tee /etc/apt/sources.list.d/docker.list > /dev/null \
+    && apt-get -q update \
+    && apt-get -q install --no-install-recommends -y \
+        binutils=* \
+        ca-certificates=* \
+        curl=* \
+        docker-buildx-plugin=* \
+        docker-ce-cli=* \
+        git=* \
+        gnupg=* \
+        rsync=* \
+        shellcheck=* \
+        wget=* \
+    && apt-get clean \
+    && rm -rf /var/lib/apt/lists/*
 
 COPY --from=hadolint /bin/hadolint /bin/hadolint
 

dev-support/jenkins/jenkins_precommit_github_yetus.sh

@@ -111,6 +111,8 @@ YETUS_ARGS+=("--tests-filter=test4tests")
 # Dockerfile since we don't want to use the auto-pulled version.
 YETUS_ARGS+=("--docker")
 YETUS_ARGS+=("--dockerfile=${DOCKERFILE}")
+# enabled docker-in-docker so that we can build container images
+YETUS_ARGS+=("--dockerind=true")
 YETUS_ARGS+=("--mvn-custom-repos")
 YETUS_ARGS+=("--java-home=${SET_JAVA_HOME}")
 # effectively treat dev-support as a custom maven module
@@ -127,4 +129,4 @@ YETUS_ARGS+=("--proclimit=5000")
 echo "Launching yetus with command line:"
 echo "${TESTPATCHBIN} ${YETUS_ARGS[*]}"
 
-/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"
+/usr/bin/env bash "${TESTPATCHBIN}" "${YETUS_ARGS[@]}"

hbase-kubernetes-deployment/components/hbase/hadoop-metrics2-hbase.properties

@@ -0,0 +1,46 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+    # syntax: [prefix].[source|sink].[instance].[options]
+    # See javadoc of package-info.java for org.apache.hadoop.metrics2 for details
+
+    *.sink.file*.class=org.apache.hadoop.metrics2.sink.FileSink
+    # default sampling period
+    *.period=10
+
+    # Below are some examples of sinks that could be used
+    # to monitor different hbase daemons.
+
+    # hbase.sink.file-all.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file-all.filename=all.metrics
+
+    # hbase.sink.file0.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file0.context=hmaster
+    # hbase.sink.file0.filename=master.metrics
+
+    # hbase.sink.file1.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file1.context=thrift-one
+    # hbase.sink.file1.filename=thrift-one.metrics
+
+    # hbase.sink.file2.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file2.context=thrift-two
+    # hbase.sink.file2.filename=thrift-one.metrics
+
+    # hbase.sink.file3.class=org.apache.hadoop.metrics2.sink.FileSink
+    # hbase.sink.file3.context=rest
+    # hbase.sink.file3.filename=rest.metrics
+
+  # Read from src rather than copy over like this.

hbase-kubernetes-deployment/components/hbase/hbase-site.xml

@@ -0,0 +1,75 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements.  See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership.  The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License.  You may obtain a copy of the License at
+
+     http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<configuration>
+  <property>
+    <name>hbase.cluster.distributed</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hbase.rootdir</name>
+    <value>hdfs://${env.HADOOP_SERVICE}:8020/hbase</value>
+  </property>
+  <property>
+    <name>hbase.zookeeper.quorum</name>
+    <value>${env.HBASE_ZOOKEEPER_QUORUM}</value>
+  </property>
+  <property>
+    <!--In k8s context, master notion of name doesn't always agree; use RS name.
+        Seems to fix case where RS was showing up w/ IP instead of name because
+        Master told it use its understanding of what its name was. See
+        HBASE-18226 and then HBASE-24667-->
+    <name>hbase.unsafe.regionserver.hostname.disable.master.reversedns</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>dfs.client.use.datanode.hostname</name>
+    <value>true</value>
+  </property>
+  <property>
+    <name>hbase.ssl.enabled</name>
+      <!-- Needs upstream fix. See https://github.com/eclipse/jetty.project/issues/4385-->
+    <value>${env.HBASE_SSL_ENABLED}</value>
+  </property>
+  <property>
+    <name>hbase.master.cleaner.interval</name>
+    <!-- Needs to clean up oldWAL and archived hfiles every 10 seconds -->
+    <value>10000</value>
+  </property>
+  <property>
+    <name>hbase.regionserver.handler.count</name>
+    <value>45</value>
+  </property>
+  <property>
+    <name>hbase.netty.eventloop.rpcserver.thread.count</name>
+    <value>1</value>
+    <description>See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default
+  is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least.</description>
+  </property>
+  <property>
+    <name>hbase.netty.worker.count</name>
+    <value>1</value>
+    <description>See the end of https://issues.apache.org/jira/browse/HBASE-27112. Default
+  is 2xCPU_COUNT which seems way too much. 1 thread seems fine for the siri workload at least.</description>
+  </property>
+  <property>
+    <name>hbase.regionserver.throughput.controller</name>
+    <value>org.apache.hadoop.hbase.regionserver.throttle.NoLimitThroughputController</value>
+  </property>
+</configuration>

hbase-kubernetes-deployment/components/hbase/kustomization.yaml

@@ -0,0 +1,40 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+---
+apiVersion: kustomize.config.k8s.io/v1alpha1
+kind: Component
+
+commonLabels:
+  app: hadoop
+
+configMapGenerator:
+- name: hbase-configuration
+  files:
+  - hbase-site.xml
+  - hadoop-metrics2-hbase.properties
+  - log4j2.properties=log4j2.properties.hbase
+- name: hbck2-configuration
+  # Add an hbck2-configuration. Uses log4j1 vs log4j2.
+  files:
+  - hbase-site.xml
+  - hadoop-metrics2-hbase.properties
+  - log4j.properties=log4j.properties.hbase
+
+resources:
+- m-service.yaml
+- m-statefulset.yaml
+- rs-service.yaml
+- rs-statefulset.yaml

hbase-kubernetes-deployment/components/hbase/log4j.properties.hbase

@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Define some default values that can be overridden by system properties
+hbase.root.logger=INFO,console
+hbase.log.dir=.
+hbase.log.file=hbase.log
+
+# Define the root logger to the system property "hbase.root.logger".
+log4j.rootLogger=${hbase.root.logger}
+
+# Logging Threshold
+log4j.threshold=ALL
+
+#
+# Daily Rolling File Appender
+#
+log4j.appender.DRFA=org.apache.log4j.DailyRollingFileAppender
+log4j.appender.DRFA.File=${hbase.log.dir}/${hbase.log.file}
+
+# Rollver at midnight
+log4j.appender.DRFA.DatePattern=.yyyy-MM-dd
+
+# 30-day backup
+#log4j.appender.DRFA.MaxBackupIndex=30
+log4j.appender.DRFA.layout=org.apache.log4j.PatternLayout
+# Debugging Pattern format
+log4j.appender.DRFA.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n
+
+
+#
+# console
+# Add "console" to rootlogger above if you want to use this
+#
+log4j.appender.console=org.apache.log4j.ConsoleAppender
+log4j.appender.console.target=System.err
+log4j.appender.console.layout=org.apache.log4j.PatternLayout
+log4j.appender.console.layout.ConversionPattern=%d{ISO8601} %-5p [%t] %C{2}(%L): %m%n
+
+# Custom Logging levels
+
+#log4j.logger.org.apache.hadoop.fs.FSNamesystem=DEBUG
+
+log4j.logger.org.apache.hadoop=WARN
+log4j.logger.org.apache.zookeeper=ERROR
+log4j.logger.org.apache.hadoop.hbase=DEBUG
+
+#These settings are workarounds against spurious logs from the minicluster.
+#See HBASE-4709
+log4j.logger.org.apache.hadoop.metrics2.impl.MetricsConfig=WARN
+log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSinkAdapter=WARN
+log4j.logger.org.apache.hadoop.metrics2.impl.MetricsSystemImpl=WARN
+log4j.logger.org.apache.hadoop.metrics2.util.MBeans=WARN
+# Enable this to get detailed connection error/retry logging.
+# log4j.logger.org.apache.hadoop.hbase.client.ConnectionImplementation=TRACE