内容纲要
概要描述
TDH 9.3.1 版本下，使用object store的姿势有别于之前的老版本，报错 无法将类 org.apache.hadoop.hbase.client.HTable中的构造器 HTable应用到给定类，原因是 HTable被淘汰，只能用Table 来实现，本文给出 demo 实例。
详细说明
1、获取Hbase api运行依赖的jar包
获取hyperbase pod中 /usr/lib/hbase/lib下所有jar包
2、windows 本地放置配置文件
从集群上获取这些配置文件，比如我们放到C盘的tmp目录下：
C:\tmp\krb5.ini
C:\tmp\hyperbase.keytab
C:\tmp\jaas.conf
C:\tmp\hbase-site.xml
C:\tmp\hdfs-site.xml
C:\tmp\core-site.xml
3、修改 jaas.conf 文件中的 keyTab 路径
4、Object Store API 示例
参考demo:
import static org.junit.Assert.assertTrue;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.*;
import org.apache.hadoop.hbase.client.*;
import org.apache.hadoop.hbase.shaded.protobuf.generated.HyperbaseProtos;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hyperbase.client.HyperbaseAdmin;
import org.apache.hadoop.hyperbase.metadata.HyperbaseMetadata;
import org.apache.hadoop.hyperbase.secondaryindex.IndexedColumn;
import org.apache.hadoop.hyperbase.secondaryindex.LOBIndex;
import org.apache.hadoop.security.UserGroupInformation;
public class objectstore {
protected static HyperbaseAdmin admin = null;
private static Connection conn;
private static final String ZK_QUORUM = "hbase.zookeeper.quorum";
private static final String ZK_CLIENT_PORT = "hbase.zookeeper.property.clientPort";
private static final String ZK_POS = "kv2:2181,kv3:2181,kv4:2181";
private static final String ZK_PORT_VALUE = "2181";
private static Configuration conf;
static {
//这里需要将集群上的/etc/krb5.conf文件放置到windows本机下，并重命名为krb5.ini
System. setProperty("java.security.krb5.conf", "C:\\tmp\\krb5.ini" );
System.setProperty("java.security.auth.login.config", "C:\\tmp\\jaas.conf");
conf = HBaseConfiguration.create();
conf.set(ZK_CLIENT_PORT, ZK_PORT_VALUE);
conf.set("hadoop.security.authentication", "kerberos");
conf.addResource(new Path("C:\\tmp\\hbase-site.xml"));
conf.addResource(new Path("C:\\tmp\\hdfs-site.xml"));
conf.addResource(new Path("C:\\tmp\\core-site.xml"));
}
public static void main(String[] args) throws Exception {
UserGroupInformation.setConfiguration(conf);
UserGroupInformation.loginUserFromKeytab("hbase/kv4@KTDH", "C:\\tmp\\hyperbase.keytab");
//创建连接池
try {
conn = ConnectionFactory.createConnection(conf);
} catch (Exception e) {
e.printStackTrace();
}
admin = new HyperbaseAdmin(conf);
testLOBGet();
}
public static void testLOBGet() throws Exception {
byte[] row = Bytes.toBytes("rowkey01");
byte[] tableName = Bytes.toBytes("SIMPLE_TEST_PUT_SCAN" + System.nanoTime());
System.out.println(tableName);
byte[] indexName = Bytes.toBytes("IDX");
byte[] family1 = Bytes.toBytes("f1");
byte[] family2 = Bytes.toBytes("f2");
createTable(TableName.valueOf(tableName), family1, family2);
addLOB(tableName, family1, indexName);
Connection connection = ConnectionFactory.createConnection(conf);
Table htable = connection.getHyperbaseTable(TableName.valueOf(tableName));
Put put = new Put(row);
String path = "C:\\tmp\\1.jpg";
byte[] value01 = getFileBytes(path);
put.addColumn(family1, Bytes.toBytes("q1"), value01);
htable.put(put);
TimeUnit.SECONDS.sleep(1);
Get get = new Get(row);
Result rs = htable.get(get);
CellScanner cs = rs.cellScanner();
while (cs.advance()) {
if(cs.current().getFamilyArray().equals("v1")){
assertTrue(Bytes.equals(CellUtil.cloneValue(cs.current()), value01));
}
System.out.println(Bytes.toString(CellUtil.cloneValue(cs.current())));
}
htable.close();
// admin.deleteTable(TableName.valueOf(tableName));
}
public static byte[] getFileBytes(String path) throws IOException {
FileInputStream fis = new FileInputStream(new File(path));// 新建一个FileInputStream对象
byte[] b = new byte[fis.available()];// 新建一个字节数组
fis.read(b);// 将文件中的内容读取到字节数组中
fis.close();
return b;
}
public static void createTable(TableName tableName, byte[]... families) throws Exception {
HTableDescriptor tableDescriptor = new HTableDescriptor(tableName);
for (byte[] family : families) {
tableDescriptor.addFamily(new HColumnDescriptor(family));
}
// 注意，object store一定要预分配region，每个region最好不要超过500G的数据。
byte[][] splitKeys = new byte[5][];
for (int i = 0; i < 5; i++) {
splitKeys[i] = Bytes.toBytes("rowkey" + i);
}
// create table succ
admin.createTable(tableDescriptor, null, splitKeys);
HyperbaseMetadata metadata = admin.getTableMetadata(tableName);
assertTrue(metadata != null);
// check metadata
assertTrue(metadata.getFulltextMetadata() == null);
assertTrue(metadata.getGlobalIndexes().isEmpty());
assertTrue(metadata.getLobs().isEmpty());
assertTrue(metadata.isTransactionTable() == false);
}
public static void addLOB(byte[] tableName, byte[] family, byte[] LOBFamily) throws IOException {
HyperbaseProtos.SecondaryIndex.Builder LOBBuilder = HyperbaseProtos.SecondaryIndex.newBuilder();
LOBBuilder.setClassName(LOBIndex.class.getName());
LOBBuilder.setUpdate(true);
LOBBuilder.setDcop(true);
IndexedColumn column = new IndexedColumn(family, Bytes.toBytes("v1"));
LOBBuilder.addColumns(column.toPb());
admin.addLob(TableName.valueOf(tableName), new LOBIndex(LOBBuilder.build()), LOBFamily,false, 2);
}
}5、查看生成的hbase表
上述代码执行成功之后，将图片已经保存在对应的hbase表中，进入hbase shell 查看生成的hbase表：
6、inceptor中创建hbase外表
--hbase.columns.mapping 是定义映射表和HBase表列之间的映射关系；
--hbase.table.name 是定义映射表对应的HBase表的表名
DROP TABLE IF EXISTS objectstore_9;
CREATE EXTERNAL TABLE objectstore_9(
key1 STRING,
ext BLOB
)
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler'
WITH SERDEPROPERTIES (
'hbase.columns.mapping'=':key,f1:q1'
)
TBLPROPERTIES (
'hbase.table.name'='SIMPLE_TEST_PUT_SCAN437876739999000');最后，我们在inceptor中查询保存的图片
SELECT * FROM objectstore_9;
FAQ:
1、如何查看代码中kerberos认证的详细报错？
建议通过修改jvm启动参数，添加-Dsun.security.krb5.debug=true的方式打印详细日志，以IDEA为例：
【Run】-【Edit Configurations...】
Add VM options:
