diff --git a/README.md b/README.md index 2ff66db39e..ff110179d4 100644 --- a/README.md +++ b/README.md @@ -23,3 +23,4 @@ operator tools including: * [HBCK2](https://github.com/apache/hbase-operator-tools/tree/master/hbase-hbck2), the hbase-2.x fix-it tool, the successor to hbase-1's _hbck_ (A.K.A _hbck1_). * [TableReporter](https://github.com/apache/hbase-operator-tools/tree/master/hbase-table-reporter), a tool to generate a basic report on Table column counts and row sizes; use when no distributed execution available. + * [MetadataRepair](https://github.com/apache/hbase-operator-tools/tree/master/hbase-meta-repair), a tool to repair hbase metadata for versions before 2.0.3 and 2.1.1 (without hbck2). diff --git a/hbase-meta-repair/README.md b/hbase-meta-repair/README.md new file mode 100644 index 0000000000..d57fa519a4 --- /dev/null +++ b/hbase-meta-repair/README.md @@ -0,0 +1,66 @@ + + +# Apache HBase Tool for repair metadata + +_MetaRepair_ is an utility tool for repairing hbase metadata table from hdfs regioninfo file. +When using the [Apache HBase™](https://hbase.apache.org) +versions before 2.0.3 and 2.1.1 (**Without HBCK2**), You can use it to fix the HBase metadata correctly. + +## Build + +Run: +``` +$ mvn install +``` +The built _hbase-meta-repair_ jar will be in the `target` sub-directory. + +## Setup +Make sure HBase tools jar is added to HBase classpath: + +``` +export HBASE_CLASSPATH=$HBASE_CLASSPATH:./hbase-meta-repair-1.1.0-SNAPSHOT.jar +``` + +## Usage + +_MetaRepair_ requires an arguments as parameters: The name of the table +to have metadata repaired. + +For example, to repair metadata of table `my-table` , assuming the _setup_ step above has been performed: + +``` +$ hbase org.apache.hbase.repair.MetaRepair my-table +``` +> The table with _namespace_ use `namespace:tablename` form of parameter. + +## Implementation Details + +HBase uses table `hbase:meta` to store metadata. The table structure is as follows: + +| Column | Description | +| ------ | ------ | +| info:state | region state | +| info:sn | region server node, It consists of server and serverstartcode, such as `slave1,16020,1557998852385` | +| info:serverstartcode | region server start timestamp | +| info:server | region server address and port，such as `slave1:16020` | +| info:seqnumDuringOpen | a binary string representing the online duration of the region | +| info:regioninfo | same as the `.regioninfo` file | + +> `info:seqnumDuringOpen` and `info:serverstartcode` will be automatically generated after the region server is restarted. + diff --git a/hbase-meta-repair/pom.xml b/hbase-meta-repair/pom.xml new file mode 100644 index 0000000000..16b5eb8e23 --- /dev/null +++ b/hbase-meta-repair/pom.xml @@ -0,0 +1,202 @@ + + + + 4.0.0 + + hbase-operator-tools + org.apache.hbase.operator.tools + 1.1.0-SNAPSHOT + .. + + + hbase-meta-repair + Apache HBase - HBase Meta Repair + Repair hbase metadata table from hdfs + + + 2.2.1 + 2.11.1 + + + + + junit + junit + 4.12 + test + + + org.apache.logging.log4j + log4j-slf4j-impl + ${log4j2.version} + + + + + + org.apache.hbase + hbase-server + ${hbase.version} + provided + + + org.apache.hbase + hbase-shaded-testing-util + ${hbase.version} + test + + + org.apache.hbase + hbase-zookeeper + ${hbase.version} + provided + test-jar + + + org.apache.hbase + hbase-common + ${hbase.version} + provided + test-jar + + + org.apache.hbase + hbase-testing-util + ${hbase.version} + test + + + org.mockito + mockito-core + 2.1.0 + test + + + + + + + src/main/resources + true + + + + + src/test/resources/META-INF/ + META-INF/ + + NOTICE + + true + + + + + org.apache.maven.plugins + maven-remote-resources-plugin + + + maven-surefire-plugin + + + + org.apache.maven.plugins + maven-source-plugin + + + org.apache.maven.plugins + maven-jar-plugin + + + + org.apache.maven.plugins + maven-shade-plugin + 3.2.0 + + + package + + shade + + + + + classworlds:classworlds + junit:junit + jmock:* + *:xml-apis + org.apache.maven:lib:tests + log4j:log4j:jar: + + + + + + + + org.apache.maven.plugins + maven-checkstyle-plugin + + true + + + + + + + + apache-release + + + + org.apache.maven.plugins + maven-resources-plugin + + + license-javadocs + prepare-package + + copy-resources + + + ${project.build.directory}/apidocs + + + src/main/javadoc/META-INF/ + META-INF/ + + NOTICE + + true + + + + + + + + + + + diff --git a/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java new file mode 100644 index 0000000000..eaba9b3f51 --- /dev/null +++ b/hbase-meta-repair/src/main/java/org/apache/hbase/repair/MetaRepair.java @@ -0,0 +1,196 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase.repair; + +import java.io.IOException; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; +import java.util.Set; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.conf.Configured; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.HBaseConfiguration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.ServerName; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Admin; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.ConnectionFactory; +import org.apache.hadoop.hbase.client.Delete; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.filter.PrefixFilter; +import org.apache.hadoop.hbase.regionserver.HRegionFileSystem; +import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; +import org.apache.hadoop.util.ToolRunner; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Repair HBase MetaData for [Apache HBase™](https://hbase.apache.org) + * versions before 2.0.3 and 2.1.1 (HBase versions without HBCK2). + */ + +public class MetaRepair extends Configured implements org.apache.hadoop.util.Tool { + + private static final Logger LOG = LoggerFactory.getLogger(MetaRepair.class.getName()); + + public static final String HBASE_META_TABLE = "hbase:meta"; + public static final String HBASE_META_FAMILY = "info"; + public static final String HBASE_META_QUALIFIER_SN = "sn"; + public static final String HBASE_META_QUALIFIER_SERVER = "server"; + public static final String HBASE_META_QUALIFIER_STATE = "state"; + + private final Configuration conf; + + public MetaRepair(Configuration conf) { + this.conf = conf; + } + + public static void main(String[] args) throws Exception { + Configuration conf = HBaseConfiguration.create(); + int errCode = ToolRunner.run(new MetaRepair(conf), args); + if (errCode != 0) { + System.exit(errCode); + } + } + + private Path getTablePath(TableName table) { + Path basePath = new Path(conf.get(HConstants.HBASE_DIR)); + basePath = new Path(basePath, "data"); + Path tablePath = new Path(basePath, table.getNamespaceAsString()); + return new Path(tablePath, table.getQualifierAsString()); + } + + /** + * Get HBase region info from the table name. + */ + public Map getMetaRegions(String tableName) throws IOException { + Connection conn = ConnectionFactory.createConnection(conf); + Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); + Scan scan = new Scan().setFilter(new PrefixFilter(Bytes.toBytes(tableName + ","))); + Map metaRegions = new HashMap<>(); + Iterator iterator = table.getScanner(scan).iterator(); + while (iterator.hasNext()) { + Result result = iterator.next(); + // Usage Bytes.toStringBinary(), Be consistent with HDFS RegionInfo encoding. + metaRegions.put(Bytes.toStringBinary(result.getRow()), result.getRow()); + } + conn.close(); + return metaRegions; + } + + /** + * Get HDFS region info from the table name. + */ + public Map getHdfsRegions(String tableName) throws IOException { + FileSystem fs = FileSystem.get(conf); + TableName table = TableName.valueOf(tableName); + Path path = this.getTablePath(table); + Map hdfsRegions = new HashMap<>(); + FileStatus[] list = fs.listStatus(path); + for (FileStatus status : list) { + if (status.isDirectory()) { + boolean isRegion = false; + FileStatus[] regions = fs.listStatus(status.getPath()); + for (FileStatus regionStatus : regions) { + // Search the .regioninfo file. + if (regionStatus.toString().contains(HRegionFileSystem.REGION_INFO_FILE)) { + isRegion = true; + break; + } + } + if (isRegion) { + // Load regioninfo file content. + RegionInfo regionInfo = HRegionFileSystem.loadRegionInfoFileContent(fs, status.getPath()); + hdfsRegions.put(regionInfo.getRegionNameAsString(), regionInfo); + } + } + } + return hdfsRegions; + } + + public void repairMetadata(String tableName) throws Exception { + Map metaRegions = getMetaRegions(tableName); + LOG.debug("HBase meta regions: {}", metaRegions.keySet()); + Map hdfsRegions = getHdfsRegions(tableName); + LOG.debug("HDFS region infos: {}", hdfsRegions.keySet()); + Set hdfsRegionNames = hdfsRegions.keySet(); + for (String hdfsRegionName : hdfsRegionNames) { + if (metaRegions.containsKey(hdfsRegionName)) { + // remove meta not in hdfs region info. + metaRegions.remove(hdfsRegionName); + } + } + Connection conn = ConnectionFactory.createConnection(conf); + Admin admin = conn.getAdmin(); + ServerName[] regionServers = admin.getRegionServers().toArray(new ServerName[0]); + LOG.info("HBase Region Servers: {}", Arrays.asList(regionServers)); + Table table = conn.getTable(TableName.valueOf(HBASE_META_TABLE)); + if (!metaRegions.isEmpty()) { + for (String regionName : metaRegions.keySet()) { + table.delete(new Delete(metaRegions.get(regionName))); + } + LOG.warn("Delete HBase Metadata: {}", metaRegions.keySet()); + } + int rsLength = regionServers.length, i = 0; + for (String regionName : hdfsRegionNames) { + String sn = regionServers[i % rsLength].getServerName(); + String[] snSig = sn.split(","); + RegionInfo regionInfo = hdfsRegions.get(regionName); + Put info = MetaTableAccessor.makePutFromRegionInfo(regionInfo, + EnvironmentEdgeManager.currentTime()); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), + Bytes.toBytes(HBASE_META_QUALIFIER_SN), Bytes.toBytes(sn)); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), + Bytes.toBytes(HBASE_META_QUALIFIER_SERVER), Bytes.toBytes(snSig[0] + ":" + snSig[1])); + info.addColumn(Bytes.toBytes(HBASE_META_FAMILY), + Bytes.toBytes(HBASE_META_QUALIFIER_STATE), Bytes.toBytes("OPEN")); + table.put(info); + i++; + } + LOG.info("Repair HBase Metadata: {}", hdfsRegionNames); + conn.close(); + } + + @Override + public int run(String[] args) { + if (args.length != 1) { + LOG.error("Wrong number of arguments. " + + "Arguments are: "); + return 1; + } + try { + this.repairMetadata(args[0]); + } catch (Exception e) { + LOG.error("Repairing metadata failed:", e); + return 2; + } + return 0; + } +} diff --git a/hbase-meta-repair/src/main/resources/log4j2.xml b/hbase-meta-repair/src/main/resources/log4j2.xml new file mode 100644 index 0000000000..9763014d23 --- /dev/null +++ b/hbase-meta-repair/src/main/resources/log4j2.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + diff --git a/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java new file mode 100644 index 0000000000..0a2f7753e1 --- /dev/null +++ b/hbase-meta-repair/src/test/java/org/apache/hbase/repair/TestMetaRepair.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hbase.repair; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +import java.io.IOException; +import java.util.Map; + +import org.apache.hadoop.hbase.HBaseTestingUtility; +import org.apache.hadoop.hbase.NamespaceDescriptor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Put; +import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.util.Bytes; +import org.junit.After; +import org.junit.AfterClass; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestMetaRepair { + + private final static HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(); + private static final String NAMESPACE = "TEST"; + private static final TableName TABLE_NAME_WITH_NAMESPACE = + TableName.valueOf(NAMESPACE, TestMetaRepair.class.getSimpleName()); + private static final TableName TABLE_NAME = + TableName.valueOf(TestMetaRepair.class.getSimpleName()); + private static final byte[] family = Bytes.toBytes("test"); + private Table table; + + @BeforeClass + public static void beforeClass() throws Exception { + TEST_UTIL.startMiniCluster(3); + } + + @AfterClass + public static void afterClass() throws Exception { + TEST_UTIL.shutdownMiniCluster(); + } + + @Before + public void setup() throws Exception { + table = TEST_UTIL.createMultiRegionTable(TABLE_NAME, family, 5); + TEST_UTIL.waitUntilAllRegionsAssigned(TABLE_NAME); + } + + @After + public void tearDown() throws Exception { + TEST_UTIL.deleteTable(TABLE_NAME); + } + + @Test + public void testHbaseAndHdfsRegions() throws Exception { + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + Map hbaseRegions = metaRepair.getMetaRegions(TABLE_NAME.getNameAsString()); + Map hdfsRegions = metaRepair.getHdfsRegions(TABLE_NAME.getNameAsString()); + assertEquals(5, hbaseRegions.size()); + assertEquals(5, hdfsRegions.size()); + assertTrue(hbaseRegions.keySet().containsAll(hdfsRegions.keySet())); + } + + @Test + public void testRepairMetadata() throws Exception { + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + generateTableData(TABLE_NAME); + final int originalCount = TEST_UTIL.countRows(table); + metaRepair.repairMetadata(TABLE_NAME.getNameAsString()); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(table)); + } + + @Test + public void testRepairMetadataWithNameSpace() throws Exception { + try { + TEST_UTIL.getAdmin().createNamespace(NamespaceDescriptor.create(NAMESPACE).build()); + Table tableWithNamespace = TEST_UTIL.createMultiRegionTable(TABLE_NAME_WITH_NAMESPACE, + family, 6); + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + final int originalCount = TEST_UTIL.countRows(tableWithNamespace); + metaRepair.repairMetadata(TABLE_NAME_WITH_NAMESPACE.getNameAsString()); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(tableWithNamespace)); + } finally { + TEST_UTIL.deleteTable(TABLE_NAME_WITH_NAMESPACE); + TEST_UTIL.getAdmin().deleteNamespace(NAMESPACE); + } + } + + @Test + public void testRepairMetadataInvalidParams() throws Exception { + final int originalCount = TEST_UTIL.countRows(table); + MetaRepair metaRepair = new MetaRepair(TEST_UTIL.getConfiguration()); + assertEquals(0, metaRepair.run(new String[]{TABLE_NAME.getNameAsString()})); + assertEquals(1, metaRepair.run(new String[]{})); + assertEquals(2, metaRepair.run(new String[]{"XXX"})); + assertEquals("Row count before and after repair should be equal", + originalCount, TEST_UTIL.countRows(table)); + } + + private void generateTableData(TableName tableName) throws Exception { + TEST_UTIL.getAdmin().getRegions(tableName).forEach(r -> { + byte[] key = r.getStartKey().length == 0 ? new byte[]{0} : r.getStartKey(); + Put put = new Put(key); + put.addColumn(family, Bytes.toBytes("c"), new byte[1024 * 1024]); + try { + table.put(put); + } catch (IOException e) { + throw new Error("Failed to put row"); + } + }); + } +} diff --git a/pom.xml b/pom.xml index 1c31a01304..4f9c1ef30c 100644 --- a/pom.xml +++ b/pom.xml @@ -59,7 +59,8 @@ hbase-operator-tools-assembly - hbase-tools + hbase-tools + hbase-meta-repair scm:git:git://gitbox.apache.org/repos/asf/hbase-operator-tools.git