如何查看HBase的HFile

記一個比較初級的筆記。

===流程===

1. 創建一張表

2. 插入10條數據

3. 查看HFile

 

===操作===

1.創建表

package api;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.io.compress.Compression;
import org.apache.hadoop.hbase.regionserver.BloomType;

public class create_table_sample1 {
    public static void main(String[] args) throws Exception {
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "192.168.1.80,192.168.1.81,192.168.1.82");
        Connection connection = ConnectionFactory.createConnection(conf);
        Admin admin = connection.getAdmin();

        HTableDescriptor desc = new HTableDescriptor(TableName.valueOf("TEST1"));
        desc.setMemStoreFlushSize(2097152L);          //2M(默認128M)
        desc.setMaxFileSize(10485760L);               //10M(默認10G)
        desc.setDurability(Durability.SYNC_WAL);   //WAL落盤方式：同步刷盤

        HColumnDescriptor family1 = new HColumnDescriptor(constants.COLUMN_FAMILY_DF.getBytes());
        family1.setTimeToLive(2 * 60 * 60 * 24);     //過期時間
        family1.setMaxVersions(2);                   //版本數
        family1.setBlockCacheEnabled(false);
        desc.addFamily(family1);
        HColumnDescriptor family2 = new HColumnDescriptor(constants.COLUMN_FAMILY_EX.getBytes());
        family2.setTimeToLive(3 * 60 * 60 * 24);     //過期時間
        family2.setMinVersions(2);                   //最小版本數
        family2.setMaxVersions(3);                   //版本數
        family2.setBloomFilterType(BloomType.ROW);   //布隆過濾方式
        family2.setBlocksize(1024);
        family2.setBlockCacheEnabled(false);
        desc.addFamily(family2);

        admin.createTable(desc);
        admin.close();
        connection.close();
    }
}

 

2.插入10條數據

package api;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.*;

import java.util.ArrayList;
import java.util.List;
import java.util.UUID;

public class table_put_sample4 {
    public static void main(String[] args) throws Exception {
        Configuration conf = HBaseConfiguration.create();
        conf.set("hbase.zookeeper.quorum", "192.168.1.80,192.168.1.81,192.168.1.82");
        conf.set("hbase.client.write.buffer", "1048576");//1M
        Connection connection = ConnectionFactory.createConnection(conf);
        BufferedMutator table = connection.getBufferedMutator(TableName.valueOf(constants.TABLE_NAME));

        List<Put> puts = new ArrayList<>();
        for(int i = 0; i < 10; i++) {
            Put put = new Put(("row" + UUID.randomUUID().toString()).getBytes());
            put.addColumn(constants.COLUMN_FAMILY_DF.getBytes(), "name".getBytes(), random.getName());
            put.addColumn(constants.COLUMN_FAMILY_DF.getBytes(), "sex".getBytes(), random.getSex());
            put.addColumn(constants.COLUMN_FAMILY_EX.getBytes(), "height".getBytes(), random.getHeight());
            put.addColumn(constants.COLUMN_FAMILY_EX.getBytes(), "weight".getBytes(), random.getWeight());
            puts.add(put);
        }

        table.mutate(puts);
        table.flush();
        table.close();
        connection.close();
    }
}

 

3. 查看HFile

命令：hbase hfile -v -p -m -f hdfs://ns/hbase/data/default/TEST1/5cd31c374a3b30bb859175495cbd6905/df/9df89dc0db7f401e943c5ded6d49d956

Scanning -> hdfs://ns/hbase/data/default/TEST1/5cd31c374a3b30bb859175495cbd6905/df/9df89dc0db7f401e943c5ded6d49d956
2017-09-29 03:53:57,233 INFO  [main] hfile.CacheConfig: Created cacheConfig: CacheConfig:disabled
K: row0324f6ce-dec9-474a-b3fd-202b0c482756/df:name/1506670800587/Put/vlen=7/seqid=8 V: wang wu
K: row0324f6ce-dec9-474a-b3fd-202b0c482756/df:sex/1506670800587/Put/vlen=3/seqid=8 V: men
K: row284986a4-66c3-4ac6-96f1-76cbf66ec0b0/df:name/1506670800410/Put/vlen=7/seqid=4 V: wei liu
K: row284986a4-66c3-4ac6-96f1-76cbf66ec0b0/df:sex/1506670800410/Put/vlen=3/seqid=4 V: men
K: row5b3796d7-0d95-4114-b8fe-15a194b87172/df:name/1506670800559/Put/vlen=5/seqid=7 V: li si
K: row5b3796d7-0d95-4114-b8fe-15a194b87172/df:sex/1506670800559/Put/vlen=3/seqid=7 V: men
K: row620c7f4b-cb20-4175-b12b-5f71349ca52e/df:name/1506670800699/Put/vlen=7/seqid=12 V: wang wu
K: row620c7f4b-cb20-4175-b12b-5f71349ca52e/df:sex/1506670800699/Put/vlen=5/seqid=12 V: women
K: row91963615-e76f-4911-be04-fcfb1e47cf64/df:name/1506670800733/Put/vlen=7/seqid=13 V: wei liu
K: row91963615-e76f-4911-be04-fcfb1e47cf64/df:sex/1506670800733/Put/vlen=5/seqid=13 V: women
K: row98e7aeea-bd63-45f3-ad28-690256303b6a/df:name/1506670800677/Put/vlen=7/seqid=11 V: wang wu
K: row98e7aeea-bd63-45f3-ad28-690256303b6a/df:sex/1506670800677/Put/vlen=3/seqid=11 V: men
K: rowa0d3ac08-188a-4869-8dcd-43cd874ae34e/df:name/1506670800476/Put/vlen=7/seqid=5 V: wang wu
K: rowa0d3ac08-188a-4869-8dcd-43cd874ae34e/df:sex/1506670800476/Put/vlen=3/seqid=5 V: men
K: rowd0584d40-bf2c-4f07-90c9-394470cc54c7/df:name/1506670800611/Put/vlen=7/seqid=9 V: wei liu
K: rowd0584d40-bf2c-4f07-90c9-394470cc54c7/df:sex/1506670800611/Put/vlen=5/seqid=9 V: women
K: rowd5e46f02-7d22-444a-a086-f0936ca81728/df:name/1506670800652/Put/vlen=7/seqid=10 V: wang wu
K: rowd5e46f02-7d22-444a-a086-f0936ca81728/df:sex/1506670800652/Put/vlen=3/seqid=10 V: men
K: rowf17bfb40-f658-4b4b-a9da-82abf455f4e6/df:name/1506670800531/Put/vlen=5/seqid=6 V: li si
K: rowf17bfb40-f658-4b4b-a9da-82abf455f4e6/df:sex/1506670800531/Put/vlen=3/seqid=6 V: men
Block index size as per heapsize: 432
reader=hdfs://ns/hbase/data/default/TEST1/5cd31c374a3b30bb859175495cbd6905/df/9df89dc0db7f401e943c5ded6d49d956,
    compression=none,
    cacheConf=CacheConfig:disabled,
    firstKey=row0324f6ce-dec9-474a-b3fd-202b0c482756/df:name/1506670800587/Put,
    lastKey=rowf17bfb40-f658-4b4b-a9da-82abf455f4e6/df:sex/1506670800531/Put,
    avgKeyLen=56,
    avgValueLen=5,
    entries=20,
    length=6440
Trailer:
    fileinfoOffset=1646,
    loadOnOpenDataOffset=1502,
    dataIndexCount=1,
    metaIndexCount=0,
    totalUncomressedBytes=6313,
    entryCount=20,
    compressionCodec=NONE,
    uncompressedDataIndexSize=70,
    numDataIndexLevels=1,
    firstDataBlockOffset=0,
    lastDataBlockOffset=0,
    comparatorClassName=org.apache.hadoop.hbase.KeyValue$KeyComparator,
    encryptionKey=NONE,
    majorVersion=3,
    minorVersion=0
Fileinfo:
    BLOOM_FILTER_TYPE = ROW
    DELETE_FAMILY_COUNT = \x00\x00\x00\x00\x00\x00\x00\x00
    EARLIEST_PUT_TS = \x00\x00\x01^\xCC\x93\xEE\x1A
    KEY_VALUE_VERSION = \x00\x00\x00\x01
    LAST_BLOOM_KEY = rowf17bfb40-f658-4b4b-a9da-82abf455f4e6
    MAJOR_COMPACTION_KEY = \x00
    MAX_MEMSTORE_TS_KEY = \x00\x00\x00\x00\x00\x00\x00\x0D
    MAX_SEQ_ID_KEY = 15
    TIMERANGE = 1506670800410....1506670800733
    hfile.AVG_KEY_LEN = 56
    hfile.AVG_VALUE_LEN = 5
    hfile.CREATE_TIME_TS = \x00\x00\x01^\xCC\x9B\xAD\xCF
    hfile.LASTKEY = \x00'rowf17bfb40-f658-4b4b-a9da-82abf455f4e6\x02dfsex\x00\x00\x01^\xCC\x93\xEE\x93\x04
Mid-key: \x00'row0324f6ce-dec9-474a-b3fd-202b0c482756\x02dfname\x00\x00\x01^\xCC\x93\xEE\xCB\x04
Bloom filter:
    BloomSize: 16
    No of Keys in bloom: 10
    Max Keys for bloom: 13
    Percentage filled: 77%
    Number of chunks: 1
    Comparator: RawBytesComparator
Delete Family Bloom filter:
    Not present
Scanned kv count -> 20

 

===Tips===：

1. HFile放在哪里了？

查看方式一：

可以通過HBase的web頁面查看HFile名稱及路徑。步驟如下：

① 打開Web管理頁面，選擇表

 

 

 ② 選擇HRegion Server

③ 選擇Region

④ 查看HFile路徑

HFile是以列族為單位的，我建立的表有兩個列族，所以這里就有兩個HFile

 

查看方式二：

直接使用hdfs命令，逐層查看

命令樣例：hadoop fs -ls /hbase/data/default

 

2. 為什么能scan到數據，卻沒有hfile？

通過程序向HBase插入數據之后，能夠scan到數據，不過hdfs上確沒有hfile。

如下圖所示：scan 'TEST1' 能夠看到表中有數據。

從Web頁面上卻看不到hfile

 

原因：

插入的數據在memstore（寫緩存）中，還沒有flush到hdfs上。

 

解決辦法：

手動flush。在hbase shell環境下，有一個flush命令，可以手動刷某張表

flush之后，就可以看到hfile了

 

--END--