From 50995ab74c3a363eb84a05aeb748a9fdd32bf1c9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 17:11:23 +0800 Subject: [PATCH 1/8] HBASE-25161 [hbase-meta-repair] Add tool for repairing metadata. --- README.md | 1 + hbase-meta-repair/README.md | 66 ++++++ hbase-meta-repair/pom.xml | 202 ++++++++++++++++++ .../org/apache/hbase/repair/MetaRepair.java | 191 +++++++++++++++++ .../src/main/resources/log4j2.xml | 35 +++ .../apache/hbase/repair/TestMetaRepair.java | 114 ++++++++++ pom.xml | 3 +- 7 files changed, 611 insertions(+), 1 deletion(-) create mode 100644 hbase-meta-repair/README.md create mode 100644 hbase-meta-repair/pom.xml create mode 100644 hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java create mode 100644 hbase-meta-repair/src/main/resources/log4j2.xml create mode 100644 hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java diff --git a/README.md b/README.md index 2ff66db39e..ff110179d4 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,4 @@ operator tools including: * [HBCK2](https://github.com/apache/hbase-operator-tools/tree/master/hbase-hbck2), the hbase-2.x fix-it tool, the successor to hbase-1's _hbck_ (A.K.A _hbck1_). * [TableReporter](https://github.com/apache/hbase-operator-tools/tree/master/hbase-table-reporter), a tool to generate a basic report on Table column counts and row sizes; use when no distributed execution available. + * [MetadataRepair](https://github.com/apache/hbase-operator-tools/tree/master/hbase-meta-repair), a tool to repair hbase metadata for versions before 2.0.3 and 2.1.1 (without hbck2). diff --git a/hbase-meta-repair/README.md b/hbase-meta-repair/README.md new file mode 100644 index 0000000000..8ec09e584e --- /dev/null +++ b/hbase-meta-repair/README.md @@ -0,0 +1,66 @@ + + +# Apache HBase Tool for repair metadata + +_MetaRepair_ is an utility tool for repairing hbase metadata table from hdfs regioninfo file. +When using the [Apache HBase™](https://hbase.apache.org) +versions before 2.0.3 and 2.1.1 (**Without HBCK2**), You can use it to fix the HBase metadata correctly. + +## Build + +Run: +``` +$ mvn install +``` +The built _hbase-meta-repair_ jar will be in the `target` sub-directory. + +## Setup +Make sure HBase tools jar is added to HBase classpath: + +``` +export HBASE_CLASSPATH=$HBASE_CLASSPATH:./hbase-meta-repair-1.1.0-SNAPSHOT.jar +``` + +## Usage + +_MetaRepair_ requires an arguments as parameters: The name of the table +to have metadata repaired. + +For example, to repair metadata of table `my-table` , assuming the _setup_ step above has been performed: + +``` +$ hbase org.apache.hbase.repair.MetaRepair my-table +``` +> The table with _namespace_ use `namespace:tablename` form of parameter. + +## Implementation Details + +HBase uses table `hbase:meta` to store metadata. The table structure is as follows: + +| Column | Description | +| ------ | ------ | +| info:state | region state | +| info:sn | region server node, It consists of server and serverstartcode, such as `slave1,16020,1557998852385` | +| info:serverstartcode | region server start timestamp | +| info:server | region server address and port，such as `slave1:16020` | +| info:seqnumDuringOpen | a binary string representing the online duration of the region | +| info:regioninfo | same as the `.regioninfo` file | + +> `info:seqnumDuringOpen` and `info:serverstartcode` will be automatically generated after the region server is restarted. + diff --git a/hbase-meta-repair/pom.xml b/hbase-meta-repair/pom.xml new file mode 100644 index 0000000000..16b5eb8e23 --- /dev/null +++ b/hbase-meta-repair/pom.xml @@ -0,0 +1,202 @@ + + + + 4.0.0 + + hbase-operator-tools + org.apache.hbase.operator.tools + 1.1.0-SNAPSHOT + .. + + + hbase-meta-repair + Apache HBase - HBase Meta Repair + Repair hbase metadata table from hdfs + + + 2.2.1 + 2.11.1 + + + + + junit + junit + 4.12 + test + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j2.version} + + + + + + org.apache.hbase + hbase-server + ${hbase.version} + provided + + + org.apache.hbase + hbase-shaded-testing-util + ${hbase.version} + test + + + org.apache.hbase + hbase-zookeeper + ${hbase.version} + provided + test-jar + + + org.apache.hbase + hbase-common + ${hbase.version} + provided + test-jar + + + org.apache.hbase + hbase-testing-util + ${hbase.version} + test + + + org.mockito + mockito-core + 2.1.0 + test + + + + + + + src/main/resources + true + + + + + src/test/resources/META-INF/ + META-INF/ + + NOTICE + + true + + + + + org.apache.maven.plugins + maven-remote-resources-plugin + + + maven-surefire-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + package + + shade + + + + + classworlds:classworlds + junit:junit + jmock:* + *:xml-apis + org.apache.maven:lib:tests + log4j:log4j:jar: + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + + + + apache-release + + + + org.apache.maven.plugins + maven-resources-plugin + + + license-javadocs + prepare-package + + copy-resources + + + ${project.build.directory}/apidocs + + + src/main/javadoc/META-INF/ + META-INF/ + + NOTICE + + true + + + + + + + + + + + diff --git a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java new file mode 100644 index 0000000000..cd208ac937 --- /dev/null +++ b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase.repair; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.*; +import org.apache.hadoop.hbase.client.*; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.util.ToolRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.*; + +/** + * Repair HBase MetaData for [Apache HBase™](https://hbase.apache.org) + * versions before 2.0.3 and 2.1.1 (HBase versions without HBCK2). + */ + +public class MetaRepair extends Configured implements org.apache.hadoop.util.Tool { + + public static final String HBASE_META_TABLE = "hbase:meta"; + public static final String HBASE_META_FAMILY = "info"; + public static final String HBASE_META_QUALIFIER_SN = "sn"; + public static final String HBASE_META_QUALIFIER_SERVER = "server"; + public static final String HBASE_META_QUALIFIER_STATE = "state"; + private static final Logger LOG = LoggerFactory.getLogger(MetaRepair.class.getName()); + private final Configuration conf; + + public MetaRepair(Configuration conf) { + this.conf = conf; + } + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + int errCode = ToolRunner.run(new MetaRepair(conf), args); + if (errCode != 0) { + System.exit(errCode); + } + } + + private Path getTablePath(TableName table) { + Path basePath = new Path(conf.get(HConstants.HBASE_DIR)); + basePath = new Path(basePath, "data"); + Path tablePath = new Path(basePath, table.getNamespaceAsString()); + return new Path(tablePath, table.getQualifierAsString()); + } + + /** + * Get HBase region info from the table name. + */ + public Map getMetaRegions(String tableName) throws IOException { + + Connection conn = ConnectionFactory.createConnection(conf); + Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); + + Scan scan = new Scan().setFilter(new PrefixFilter(Bytes.toBytes(tableName + ","))); + + Map metaRegions = new HashMap<>(); + + Iterator iterator = table.getScanner(scan).iterator(); + while (iterator.hasNext()) { + Result result = iterator.next(); + // Usage Bytes.toStringBinary(), Be consistent with HDFS RegionInfo encoding. + metaRegions.put(Bytes.toStringBinary(result.getRow()), result.getRow()); + } + conn.close(); + + return metaRegions; + } + + /** + * Get HDFS region info from the table name. + */ + public Map getHdfsRegions(String tableName) throws IOException { + + FileSystem fs = FileSystem.get(conf); + TableName table = TableName.valueOf(tableName); + Path path = this.getTablePath(table); + + Map hdfsRegions = new HashMap<>(); + + FileStatus[] list = fs.listStatus(path); + for (FileStatus status : list) { + if (status.isDirectory()) { + boolean isRegion = false; + FileStatus[] regions = fs.listStatus(status.getPath()); + for (FileStatus regionStatus : regions) { + // Search the .regioninfo file. + if (regionStatus.toString().contains(HRegionFileSystem.REGION_INFO_FILE)) { + isRegion = true; + break; + } + } + if (isRegion) { + // Load regioninfo file content. + RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, status.getPath()); + hdfsRegions.put(regionInfo.getRegionNameAsString(), regionInfo); + } + } + } + return hdfsRegions; + } + + public void repairMetadata(String tableName) throws Exception { + + Map metaRegions = getMetaRegions(tableName); + LOG.debug("HBase meta regions: {}", metaRegions.keySet()); + + Map hdfsRegions = getHdfsRegions(tableName); + LOG.debug("HDFS region infos: {}", hdfsRegions.keySet()); + + Set hdfsRegionNames = hdfsRegions.keySet(); + for (String hdfsRegionName : hdfsRegionNames) { + if (metaRegions.containsKey(hdfsRegionName)) { + // remove meta not in hdfs region info. + metaRegions.remove(hdfsRegionName); + } + } + Connection conn = ConnectionFactory.createConnection(conf); + Admin admin = conn.getAdmin(); + + ServerName[] regionServers = admin.getRegionServers().toArray(new ServerName[0]); + LOG.info("HBase Region Servers: {}", Arrays.asList(regionServers)); + + Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); + + if (!metaRegions.isEmpty()) { + for (String regionName : metaRegions.keySet()) { + table.delete(new Delete(metaRegions.get(regionName))); + } + LOG.warn("Delete HBase Metadata: {}", metaRegions.keySet()); + } + + int rsLength = regionServers.length, i = 0; + + for (String regionName : hdfsRegionNames) { + String sn = regionServers[i % rsLength].getServerName(); + String[] snSig = sn.split(","); + RegionInfo regionInfo = hdfsRegions.get(regionName); + Put info = MetaTableAccessor.makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime()); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), Bytes.toBytes(HBASE_META_QUALIFIER_SN), Bytes.toBytes(sn)); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), Bytes.toBytes(HBASE_META_QUALIFIER_SERVER), Bytes.toBytes(snSig[0] + ":" + snSig[1])); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), Bytes.toBytes(HBASE_META_QUALIFIER_STATE), Bytes.toBytes("OPEN")); + table.put(info); + i++; + } + LOG.info("Repair HBase Metadata: {}", hdfsRegionNames); + + conn.close(); + } + + @Override + public int run(String[] args) { + if (args.length != 1) { + LOG.error("Wrong number of arguments. " + + "Arguments are: "); + return 1; + } + try { + this.repairMetadata(args[0]); + } catch (Exception e) { + LOG.error("Repairing metadata failed:", e); + return 2; + } + return 0; + } +} diff --git a/hbase-meta-repair/src/main/resources/log4j2.xml b/hbase-meta-repair/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..9763014d23 --- /dev/null +++ b/hbase-meta-repair/src/main/resources/log4j2.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java new file mode 100644 index 0000000000..3fbdff5ea9 --- /dev/null +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -0,0 +1,114 @@ +package org.apache.hbase.repair; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.*; + +import java.io.IOException; +import java.util.Map; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +public class TestMetaRepair { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String NAMESPACE = "TEST"; + private static final TableName TABLE_NAME_WITH_NAMESPACE = + TableName.valueOf(NAMESPACE, TestMetaRepair.class.getSimpleName()); + private static final TableName TABLE_NAME = + TableName.valueOf(TestMetaRepair.class.getSimpleName()); + private static final byte[] family = Bytes.toBytes("test"); + private Table table; + + @BeforeClass + public static void beforeClass() throws Exception { + TEST_UTIL.getConfiguration().set(HConstants.HREGION_MAX_FILESIZE, + Long.toString(1024 * 1024 * 3)); + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void afterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup() throws Exception { + table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, family, 10); + TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.deleteTable(TABLE_NAME); + } + + @Test + public void testHbaseAndHdfsRegions() throws Exception { + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + Map hbaseRegions = metaRepair.getMetaRegions(TABLE_NAME.getNameAsString()); + Map hdfsRegions = metaRepair.getHdfsRegions(TABLE_NAME.getNameAsString()); + assertEquals(10, hbaseRegions.size()); + assertEquals(10, hdfsRegions.size()); + assertTrue(hbaseRegions.keySet().containsAll(hdfsRegions.keySet())); + } + + @Test + public void testRepairMetadata() throws Exception { + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + generateTableData(TABLE_NAME); + final int originalCount = TEST_UTIL.countRows(table); + metaRepair.repairMetadata(TABLE_NAME.getNameAsString()); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(table)); + } + + @Test + public void testRepairMetadataWithNameSpace() throws Exception { + try { + TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); + Table tableWithNamespace = TEST_UTIL.createMultiRegionTable(TABLE_NAME_WITH_NAMESPACE, family, 6); + TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME_WITH_NAMESPACE); + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + generateTableData(TABLE_NAME_WITH_NAMESPACE); + final int originalCount = TEST_UTIL.countRows(tableWithNamespace); + metaRepair.repairMetadata(TABLE_NAME_WITH_NAMESPACE.getNameAsString()); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(tableWithNamespace)); + } finally { + TEST_UTIL.deleteTable(TABLE_NAME_WITH_NAMESPACE); + TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE); + } + } + + @Test + public void testRepairMetadataInvalidParams() throws Exception { + final int originalCount = TEST_UTIL.countRows(table); + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + assertEquals(0, metaRepair.run(new String[]{TABLE_NAME.getNameAsString()})); + assertEquals(1, metaRepair.run(new String[]{})); + assertEquals(2, metaRepair.run(new String[]{"XXX"})); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(table)); + } + + private void generateTableData(TableName tableName) throws Exception { + TEST_UTIL.getAdmin().getRegions(tableName).forEach(r -> { + byte[] key = r.getStartKey().length == 0 ? new byte[]{0} : r.getStartKey(); + Put put = new Put(key); + put.addColumn(family, Bytes.toBytes("c"), new byte[1024 * 1024]); + try { + table.put(put); + } catch (IOException e) { + throw new Error("Failed to put row"); + } + }); + } +} diff --git a/pom.xml b/pom.xml index 1c31a01304..4f9c1ef30c 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,8 @@ hbase-operator-tools-assembly - hbase-tools + hbase-tools + hbase-meta-repair scm:git:git://gitbox.apache.org/repos/asf/hbase-operator-tools.git From 7e9dd0b043dc102ccc5dea724f1266e96981e76e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 18:23:54 +0800 Subject: [PATCH 2/8] HBASE-25161 [hbase-meta-repair] fix code style. --- hbase-meta-repair/README.md | 20 +- .../org/apache/hbase/repair/MetaRepair.java | 289 +++++++++--------- .../apache/hbase/repair/TestMetaRepair.java | 200 ++++++------ 3 files changed, 267 insertions(+), 242 deletions(-) diff --git a/hbase-meta-repair/README.md b/hbase-meta-repair/README.md index 8ec09e584e..d57fa519a4 100644 --- a/hbase-meta-repair/README.md +++ b/hbase-meta-repair/README.md @@ -18,8 +18,8 @@ # Apache HBase Tool for repair metadata -_MetaRepair_ is an utility tool for repairing hbase metadata table from hdfs regioninfo file. -When using the [Apache HBase™](https://hbase.apache.org) +_MetaRepair_ is an utility tool for repairing hbase metadata table from hdfs regioninfo file. +When using the [Apache HBase™](https://hbase.apache.org) versions before 2.0.3 and 2.1.1 (**Without HBCK2**), You can use it to fix the HBase metadata correctly. ## Build @@ -47,20 +47,20 @@ For example, to repair metadata of table `my-table` , assuming the _setup_ step ``` $ hbase org.apache.hbase.repair.MetaRepair my-table ``` -> The table with _namespace_ use `namespace:tablename` form of parameter. +> The table with _namespace_ use `namespace:tablename` form of parameter. ## Implementation Details HBase uses table `hbase:meta` to store metadata. The table structure is as follows: -| Column | Description | +| Column | Description | | ------ | ------ | -| info:state | region state | -| info:sn | region server node, It consists of server and serverstartcode, such as `slave1,16020,1557998852385` | -| info:serverstartcode | region server start timestamp | -| info:server | region server address and port，such as `slave1:16020` | -| info:seqnumDuringOpen | a binary string representing the online duration of the region | -| info:regioninfo | same as the `.regioninfo` file | +| info:state | region state | +| info:sn | region server node, It consists of server and serverstartcode, such as `slave1,16020,1557998852385` | +| info:serverstartcode | region server start timestamp | +| info:server | region server address and port，such as `slave1:16020` | +| info:seqnumDuringOpen | a binary string representing the online duration of the region | +| info:regioninfo | same as the `.regioninfo` file | > `info:seqnumDuringOpen` and `info:serverstartcode` will be automatically generated after the region server is restarted. diff --git a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java index cd208ac937..b36f6f05e9 100644 --- a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java +++ b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java @@ -17,24 +17,40 @@ */ package org.apache.hbase.repair; +import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; +import java.util.Map; +import java.util.HashMap; +import java.util.Set; + import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.*; -import org.apache.hadoop.hbase.client.*; -import org.apache.hadoop.hbase.filter.PrefixFilter; import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.ToolRunner; import org.slf4j.Logger; import org.slf4j.LoggerFactory; -import java.io.IOException; -import java.util.*; - /** * Repair HBase MetaData for [Apache HBase™](https://hbase.apache.org) * versions before 2.0.3 and 2.1.1 (HBase versions without HBCK2). @@ -42,150 +58,139 @@ public class MetaRepair extends Configured implements org.apache.hadoop.util.Tool { - public static final String HBASE_META_TABLE = "hbase:meta"; - public static final String HBASE_META_FAMILY = "info"; - public static final String HBASE_META_QUALIFIER_SN = "sn"; - public static final String HBASE_META_QUALIFIER_SERVER = "server"; - public static final String HBASE_META_QUALIFIER_STATE = "state"; - private static final Logger LOG = LoggerFactory.getLogger(MetaRepair.class.getName()); - private final Configuration conf; - - public MetaRepair(Configuration conf) { - this.conf = conf; - } - - public static void main(String[] args) throws Exception { - Configuration conf = HBaseConfiguration.create(); - int errCode = ToolRunner.run(new MetaRepair(conf), args); - if (errCode != 0) { - System.exit(errCode); - } - } - - private Path getTablePath(TableName table) { - Path basePath = new Path(conf.get(HConstants.HBASE_DIR)); - basePath = new Path(basePath, "data"); - Path tablePath = new Path(basePath, table.getNamespaceAsString()); - return new Path(tablePath, table.getQualifierAsString()); - } - - /** - * Get HBase region info from the table name. - */ - public Map getMetaRegions(String tableName) throws IOException { - - Connection conn = ConnectionFactory.createConnection(conf); - Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); + private static final Logger LOG = LoggerFactory.getLogger(MetaRepair.class.getName()); - Scan scan = new Scan().setFilter(new PrefixFilter(Bytes.toBytes(tableName + ","))); + public static final String HBASE_META_TABLE = "hbase:meta"; + public static final String HBASE_META_FAMILY = "info"; + public static final String HBASE_META_QUALIFIER_SN = "sn"; + public static final String HBASE_META_QUALIFIER_SERVER = "server"; + public static final String HBASE_META_QUALIFIER_STATE = "state"; - Map metaRegions = new HashMap<>(); + private final Configuration conf; - Iterator iterator = table.getScanner(scan).iterator(); - while (iterator.hasNext()) { - Result result = iterator.next(); - // Usage Bytes.toStringBinary(), Be consistent with HDFS RegionInfo encoding. - metaRegions.put(Bytes.toStringBinary(result.getRow()), result.getRow()); - } - conn.close(); + public MetaRepair(Configuration conf) { + this.conf = conf; + } - return metaRegions; + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + int errCode = ToolRunner.run(new MetaRepair(conf), args); + if (errCode != 0) { + System.exit(errCode); } - - /** - * Get HDFS region info from the table name. - */ - public Map getHdfsRegions(String tableName) throws IOException { - - FileSystem fs = FileSystem.get(conf); - TableName table = TableName.valueOf(tableName); - Path path = this.getTablePath(table); - - Map hdfsRegions = new HashMap<>(); - - FileStatus[] list = fs.listStatus(path); - for (FileStatus status : list) { - if (status.isDirectory()) { - boolean isRegion = false; - FileStatus[] regions = fs.listStatus(status.getPath()); - for (FileStatus regionStatus : regions) { - // Search the .regioninfo file. - if (regionStatus.toString().contains(HRegionFileSystem.REGION_INFO_FILE)) { - isRegion = true; - break; - } - } - if (isRegion) { - // Load regioninfo file content. - RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, status.getPath()); - hdfsRegions.put(regionInfo.getRegionNameAsString(), regionInfo); - } - } - } - return hdfsRegions; + } + + private Path getTablePath(TableName table) { + Path basePath = new Path(conf.get(HConstants.HBASE_DIR)); + basePath = new Path(basePath, "data"); + Path tablePath = new Path(basePath, table.getNamespaceAsString()); + return new Path(tablePath, table.getQualifierAsString()); + } + + /** + * Get HBase region info from the table name. + */ + public Map getMetaRegions(String tableName) throws IOException { + Connection conn = ConnectionFactory.createConnection(conf); + Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); + Scan scan = new Scan().setFilter(new PrefixFilter(Bytes.toBytes(tableName + ","))); + Map metaRegions = new HashMap<>(); + Iterator iterator = table.getScanner(scan).iterator(); + while (iterator.hasNext()) { + Result result = iterator.next(); + // Usage Bytes.toStringBinary(), Be consistent with HDFS RegionInfo encoding. + metaRegions.put(Bytes.toStringBinary(result.getRow()), result.getRow()); } - - public void repairMetadata(String tableName) throws Exception { - - Map metaRegions = getMetaRegions(tableName); - LOG.debug("HBase meta regions: {}", metaRegions.keySet()); - - Map hdfsRegions = getHdfsRegions(tableName); - LOG.debug("HDFS region infos: {}", hdfsRegions.keySet()); - - Set hdfsRegionNames = hdfsRegions.keySet(); - for (String hdfsRegionName : hdfsRegionNames) { - if (metaRegions.containsKey(hdfsRegionName)) { - // remove meta not in hdfs region info. - metaRegions.remove(hdfsRegionName); - } + conn.close(); + return metaRegions; + } + + /** + * Get HDFS region info from the table name. + */ + public Map getHdfsRegions(String tableName) throws IOException { + FileSystem fs = FileSystem.get(conf); + TableName table = TableName.valueOf(tableName); + Path path = this.getTablePath(table); + Map hdfsRegions = new HashMap<>(); + FileStatus[] list = fs.listStatus(path); + for (FileStatus status : list) { + if (status.isDirectory()) { + boolean isRegion = false; + FileStatus[] regions = fs.listStatus(status.getPath()); + for (FileStatus regionStatus : regions) { + // Search the .regioninfo file. + if (regionStatus.toString().contains(HRegionFileSystem.REGION_INFO_FILE)) { + isRegion = true; + break; + } } - Connection conn = ConnectionFactory.createConnection(conf); - Admin admin = conn.getAdmin(); - - ServerName[] regionServers = admin.getRegionServers().toArray(new ServerName[0]); - LOG.info("HBase Region Servers: {}", Arrays.asList(regionServers)); - - Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); - - if (!metaRegions.isEmpty()) { - for (String regionName : metaRegions.keySet()) { - table.delete(new Delete(metaRegions.get(regionName))); - } - LOG.warn("Delete HBase Metadata: {}", metaRegions.keySet()); + if (isRegion) { + // Load regioninfo file content. + RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, status.getPath()); + hdfsRegions.put(regionInfo.getRegionNameAsString(), regionInfo); } - - int rsLength = regionServers.length, i = 0; - - for (String regionName : hdfsRegionNames) { - String sn = regionServers[i % rsLength].getServerName(); - String[] snSig = sn.split(","); - RegionInfo regionInfo = hdfsRegions.get(regionName); - Put info = MetaTableAccessor.makePutFromRegionInfo(regionInfo, EnvironmentEdgeManager.currentTime()); - info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), Bytes.toBytes(HBASE_META_QUALIFIER_SN), Bytes.toBytes(sn)); - info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), Bytes.toBytes(HBASE_META_QUALIFIER_SERVER), Bytes.toBytes(snSig[0] + ":" + snSig[1])); - info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), Bytes.toBytes(HBASE_META_QUALIFIER_STATE), Bytes.toBytes("OPEN")); - table.put(info); - i++; - } - LOG.info("Repair HBase Metadata: {}", hdfsRegionNames); - - conn.close(); + } } - - @Override - public int run(String[] args) { - if (args.length != 1) { - LOG.error("Wrong number of arguments. " - + "Arguments are: "); - return 1; - } - try { - this.repairMetadata(args[0]); - } catch (Exception e) { - LOG.error("Repairing metadata failed:", e); - return 2; - } - return 0; + return hdfsRegions; + } + + public void repairMetadata(String tableName) throws Exception { + Map metaRegions = getMetaRegions(tableName); + LOG.debug("HBase meta regions: {}", metaRegions.keySet()); + Map hdfsRegions = getHdfsRegions(tableName); + LOG.debug("HDFS region infos: {}", hdfsRegions.keySet()); + Set hdfsRegionNames = hdfsRegions.keySet(); + for (String hdfsRegionName : hdfsRegionNames) { + if (metaRegions.containsKey(hdfsRegionName)) { + // remove meta not in hdfs region info. + metaRegions.remove(hdfsRegionName); + } + } + Connection conn = ConnectionFactory.createConnection(conf); + Admin admin = conn.getAdmin(); + ServerName[] regionServers = admin.getRegionServers().toArray(new ServerName[0]); + LOG.info("HBase Region Servers: {}", Arrays.asList(regionServers)); + Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); + if (!metaRegions.isEmpty()) { + for (String regionName : metaRegions.keySet()) { + table.delete(new Delete(metaRegions.get(regionName))); + } + LOG.warn("Delete HBase Metadata: {}", metaRegions.keySet()); + } + int rsLength = regionServers.length, i = 0; + for (String regionName : hdfsRegionNames) { + String sn = regionServers[i % rsLength].getServerName(); + String[] snSig = sn.split(","); + RegionInfo regionInfo = hdfsRegions.get(regionName); + Put info = MetaTableAccessor.makePutFromRegionInfo(regionInfo, + EnvironmentEdgeManager.currentTime()); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), + Bytes.toBytes(HBASE_META_QUALIFIER_SN), Bytes.toBytes(sn)); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), + Bytes.toBytes(HBASE_META_QUALIFIER_SERVER), Bytes.toBytes(snSig[0] + ":" + snSig[1])); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), + Bytes.toBytes(HBASE_META_QUALIFIER_STATE), Bytes.toBytes("OPEN")); + table.put(info); + i++; + } + LOG.info("Repair HBase Metadata: {}", hdfsRegionNames); + conn.close(); + } + + @Override + public int run(String[] args) { + if (args.length != 1) { + LOG.error("Wrong number of arguments. " + + "Arguments are: "); + return 1; + } + try { + this.repairMetadata(args[0]); + } catch (Exception e) { + LOG.error("Repairing metadata failed:", e); + return 2; } + return 0; + } } diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java index 3fbdff5ea9..8c52a50820 100644 --- a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -1,5 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ package org.apache.hbase.repair; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map; + import org.apache.hadoop.hbase.HBaseTestingUtility; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.NamespaceDescriptor; @@ -8,107 +31,104 @@ import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.util.Bytes; -import org.junit.*; - -import java.io.IOException; -import java.util.Map; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; public class TestMetaRepair { - private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); - private static final String NAMESPACE = "TEST"; - private static final TableName TABLE_NAME_WITH_NAMESPACE = - TableName.valueOf(NAMESPACE, TestMetaRepair.class.getSimpleName()); - private static final TableName TABLE_NAME = - TableName.valueOf(TestMetaRepair.class.getSimpleName()); - private static final byte[] family = Bytes.toBytes("test"); - private Table table; + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String NAMESPACE = "TEST"; + private static final TableName TABLE_NAME_WITH_NAMESPACE = + TableName.valueOf(NAMESPACE, TestMetaRepair.class.getSimpleName()); + private static final TableName TABLE_NAME = + TableName.valueOf(TestMetaRepair.class.getSimpleName()); + private static final byte[] family = Bytes.toBytes("test"); + private Table table; - @BeforeClass - public static void beforeClass() throws Exception { - TEST_UTIL.getConfiguration().set(HConstants.HREGION_MAX_FILESIZE, - Long.toString(1024 * 1024 * 3)); - TEST_UTIL.startMiniCluster(3); - } + @BeforeClass + public static void beforeClass() throws Exception { + TEST_UTIL.getConfiguration().set(HConstants.HREGION_MAX_FILESIZE, Long.toString(1024 * 1024 * 1)); + TEST_UTIL.startMiniCluster(3); + } - @AfterClass - public static void afterClass() throws Exception { - TEST_UTIL.shutdownMiniCluster(); - } + @AfterClass + public static void afterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } - @Before - public void setup() throws Exception { - table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, family, 10); - TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); - } + @Before + public void setup() throws Exception { + table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, family, 10); + TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); + } - @After - public void tearDown() throws Exception { - TEST_UTIL.deleteTable(TABLE_NAME); - } + @After + public void tearDown() throws Exception { + TEST_UTIL.deleteTable(TABLE_NAME); + } - @Test - public void testHbaseAndHdfsRegions() throws Exception { - MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); - Map hbaseRegions = metaRepair.getMetaRegions(TABLE_NAME.getNameAsString()); - Map hdfsRegions = metaRepair.getHdfsRegions(TABLE_NAME.getNameAsString()); - assertEquals(10, hbaseRegions.size()); - assertEquals(10, hdfsRegions.size()); - assertTrue(hbaseRegions.keySet().containsAll(hdfsRegions.keySet())); - } + @Test + public void testHbaseAndHdfsRegions() throws Exception { + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + Map hbaseRegions = metaRepair.getMetaRegions(TABLE_NAME.getNameAsString()); + Map hdfsRegions = metaRepair.getHdfsRegions(TABLE_NAME.getNameAsString()); + assertEquals(10, hbaseRegions.size()); + assertEquals(10, hdfsRegions.size()); + assertTrue(hbaseRegions.keySet().containsAll(hdfsRegions.keySet())); + } - @Test - public void testRepairMetadata() throws Exception { - MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); - generateTableData(TABLE_NAME); - final int originalCount = TEST_UTIL.countRows(table); - metaRepair.repairMetadata(TABLE_NAME.getNameAsString()); - assertEquals("Row count before and after repair should be equal", - originalCount, TEST_UTIL.countRows(table)); - } + @Test + public void testRepairMetadata() throws Exception { + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + generateTableData(TABLE_NAME); + final int originalCount = TEST_UTIL.countRows(table); + metaRepair.repairMetadata(TABLE_NAME.getNameAsString()); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(table)); + } - @Test - public void testRepairMetadataWithNameSpace() throws Exception { - try { - TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); - Table tableWithNamespace = TEST_UTIL.createMultiRegionTable(TABLE_NAME_WITH_NAMESPACE, family, 6); - TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME_WITH_NAMESPACE); - MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); - generateTableData(TABLE_NAME_WITH_NAMESPACE); - final int originalCount = TEST_UTIL.countRows(tableWithNamespace); - metaRepair.repairMetadata(TABLE_NAME_WITH_NAMESPACE.getNameAsString()); - assertEquals("Row count before and after repair should be equal", - originalCount, TEST_UTIL.countRows(tableWithNamespace)); - } finally { - TEST_UTIL.deleteTable(TABLE_NAME_WITH_NAMESPACE); - TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE); - } + @Test + public void testRepairMetadataWithNameSpace() throws Exception { + try { + TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); + Table tableWithNamespace = TEST_UTIL.createMultiRegionTable(TABLE_NAME_WITH_NAMESPACE, family, 6); + TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME_WITH_NAMESPACE); + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + generateTableData(TABLE_NAME_WITH_NAMESPACE); + final int originalCount = TEST_UTIL.countRows(tableWithNamespace); + metaRepair.repairMetadata(TABLE_NAME_WITH_NAMESPACE.getNameAsString()); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(tableWithNamespace)); + } finally { + TEST_UTIL.deleteTable(TABLE_NAME_WITH_NAMESPACE); + TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE); } + } - @Test - public void testRepairMetadataInvalidParams() throws Exception { - final int originalCount = TEST_UTIL.countRows(table); - MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); - assertEquals(0, metaRepair.run(new String[]{TABLE_NAME.getNameAsString()})); - assertEquals(1, metaRepair.run(new String[]{})); - assertEquals(2, metaRepair.run(new String[]{"XXX"})); - assertEquals("Row count before and after repair should be equal", - originalCount, TEST_UTIL.countRows(table)); - } + @Test + public void testRepairMetadataInvalidParams() throws Exception { + final int originalCount = TEST_UTIL.countRows(table); + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + assertEquals(0, metaRepair.run(new String[]{TABLE_NAME.getNameAsString()})); + assertEquals(1, metaRepair.run(new String[]{})); + assertEquals(2, metaRepair.run(new String[]{"XXX"})); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(table)); + } - private void generateTableData(TableName tableName) throws Exception { - TEST_UTIL.getAdmin().getRegions(tableName).forEach(r -> { - byte[] key = r.getStartKey().length == 0 ? new byte[]{0} : r.getStartKey(); - Put put = new Put(key); - put.addColumn(family, Bytes.toBytes("c"), new byte[1024 * 1024]); - try { - table.put(put); - } catch (IOException e) { - throw new Error("Failed to put row"); - } - }); - } + private void generateTableData(TableName tableName) throws Exception { + TEST_UTIL.getAdmin().getRegions(tableName).forEach(r -> { + byte[] key = r.getStartKey().length == 0 ? new byte[]{0} : r.getStartKey(); + Put put = new Put(key); + put.addColumn(family, Bytes.toBytes("c"), new byte[1024 * 1024]); + try { + table.put(put); + } catch (IOException e) { + throw new Error("Failed to put row"); + } + }); + } } From 9dfcb16f4a929e6c385c17e1e0d0f4ae92dddeac Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 18:32:46 +0800 Subject: [PATCH 3/8] HBASE-25161 [hbase-meta-repair] fix code style. --- .../src/test/java/org/apache/hbase/repair/TestMetaRepair.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java index 8c52a50820..0560ace3a6 100644 --- a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -42,9 +42,9 @@ public class TestMetaRepair { private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); private static final String NAMESPACE = "TEST"; private static final TableName TABLE_NAME_WITH_NAMESPACE = - TableName.valueOf(NAMESPACE, TestMetaRepair.class.getSimpleName()); + TableName.valueOf(NAMESPACE, TestMetaRepair.class.getSimpleName()); private static final TableName TABLE_NAME = - TableName.valueOf(TestMetaRepair.class.getSimpleName()); + TableName.valueOf(TestMetaRepair.class.getSimpleName()); private static final byte[] family = Bytes.toBytes("test"); private Table table; From e582d26737e20bfd6f649c4496eb03ba73522592 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 18:36:10 +0800 Subject: [PATCH 4/8] HBASE-25161 [hbase-meta-repair] fix testcase issue. --- .../src/test/java/org/apache/hbase/repair/TestMetaRepair.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java index 0560ace3a6..baf5246d6d 100644 --- a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -95,9 +95,7 @@ public void testRepairMetadataWithNameSpace() throws Exception { try { TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); Table tableWithNamespace = TEST_UTIL.createMultiRegionTable(TABLE_NAME_WITH_NAMESPACE, family, 6); - TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME_WITH_NAMESPACE); MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); - generateTableData(TABLE_NAME_WITH_NAMESPACE); final int originalCount = TEST_UTIL.countRows(tableWithNamespace); metaRepair.repairMetadata(TABLE_NAME_WITH_NAMESPACE.getNameAsString()); assertEquals("Row count before and after repair should be equal", From 8d9358a4e7345324d0c7d71371eab0c00bc54087 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 18:43:54 +0800 Subject: [PATCH 5/8] HBASE-25161 [hbase-meta-repair] fix code style. --- .../main/java/org/apache/hbase/repair/MetaRepair.java | 10 +++++----- .../java/org/apache/hbase/repair/TestMetaRepair.java | 6 ++++-- 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java index b36f6f05e9..45a07b552d 100644 --- a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java +++ b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java @@ -19,9 +19,9 @@ import java.io.IOException; import java.util.Arrays; +import java.util.HashMap; import java.util.Iterator; import java.util.Map; -import java.util.HashMap; import java.util.Set; import org.apache.hadoop.conf.Configuration; @@ -29,22 +29,22 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; -import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Delete; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.PrefixFilter; -import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; import org.apache.hadoop.hbase.util.Bytes; import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; import org.apache.hadoop.util.ToolRunner; diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java index baf5246d6d..b38c2f4b52 100644 --- a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -50,7 +50,8 @@ public class TestMetaRepair { @BeforeClass public static void beforeClass() throws Exception { - TEST_UTIL.getConfiguration().set(HConstants.HREGION_MAX_FILESIZE, Long.toString(1024 * 1024 * 1)); + TEST_UTIL.getConfiguration().set(HConstants.HREGION_MAX_FILESIZE, + Long.toString(1024 * 1024 * 1)); TEST_UTIL.startMiniCluster(3); } @@ -94,7 +95,8 @@ public void testRepairMetadata() throws Exception { public void testRepairMetadataWithNameSpace() throws Exception { try { TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); - Table tableWithNamespace = TEST_UTIL.createMultiRegionTable(TABLE_NAME_WITH_NAMESPACE, family, 6); + Table tableWithNamespace = TEST_UTIL.createMultiRegionTable(TABLE_NAME_WITH_NAMESPACE, + family, 6); MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); final int originalCount = TEST_UTIL.countRows(tableWithNamespace); metaRepair.repairMetadata(TABLE_NAME_WITH_NAMESPACE.getNameAsString()); From ffe3aa535dd02e3c02705c7d1a0b0b2ae5d218f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 18:45:48 +0800 Subject: [PATCH 6/8] HBASE-25161 [hbase-meta-repair] fix testcase issue. --- .../src/test/java/org/apache/hbase/repair/TestMetaRepair.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java index b38c2f4b52..d4915ba0a2 100644 --- a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -51,7 +51,7 @@ public class TestMetaRepair { @BeforeClass public static void beforeClass() throws Exception { TEST_UTIL.getConfiguration().set(HConstants.HREGION_MAX_FILESIZE, - Long.toString(1024 * 1024 * 1)); + Long.toString(1024 * 1024 * 3)); TEST_UTIL.startMiniCluster(3); } From ac7b00eec07f9267f11ce1e55a75e77fc6909472 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 19:47:03 +0800 Subject: [PATCH 7/8] HBASE-25161 [hbase-meta-repair] fix code style. --- .../src/main/java/org/apache/hbase/repair/MetaRepair.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java index 45a07b552d..eaba9b3f51 100644 --- a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java +++ b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java @@ -31,16 +31,16 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.ServerName; import org.apache.hadoop.hbase.TableName; -import org.apache.hadoop.hbase.MetaTableAccessor; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.client.Connection; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.RegionInfo; import org.apache.hadoop.hbase.client.Result; -import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.filter.PrefixFilter; From afe272df81351b2e4f25dc9bc498535877ce9af5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E4=B8=9A=E4=BD=99=E5=B8=83=E9=81=93=E5=B8=88?= Date: Wed, 7 Oct 2020 19:52:13 +0800 Subject: [PATCH 8/8] HBASE-25161 [hbase-meta-repair] fix testcase issue. --- .../java/org/apache/hbase/repair/TestMetaRepair.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java index d4915ba0a2..0a2f7753e1 100644 --- a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -24,7 +24,6 @@ import java.util.Map; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.NamespaceDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Put; @@ -50,8 +49,6 @@ public class TestMetaRepair { @BeforeClass public static void beforeClass() throws Exception { - TEST_UTIL.getConfiguration().set(HConstants.HREGION_MAX_FILESIZE, - Long.toString(1024 * 1024 * 3)); TEST_UTIL.startMiniCluster(3); } @@ -62,7 +59,7 @@ public static void afterClass() throws Exception { @Before public void setup() throws Exception { - table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, family, 10); + table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, family, 5); TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); } @@ -76,8 +73,8 @@ public void testHbaseAndHdfsRegions() throws Exception { MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); Map hbaseRegions = metaRepair.getMetaRegions(TABLE_NAME.getNameAsString()); Map hdfsRegions = metaRepair.getHdfsRegions(TABLE_NAME.getNameAsString()); - assertEquals(10, hbaseRegions.size()); - assertEquals(10, hdfsRegions.size()); + assertEquals(5, hbaseRegions.size()); + assertEquals(5, hdfsRegions.size()); assertTrue(hbaseRegions.keySet().containsAll(hdfsRegions.keySet())); }