一、過濾器(Filter)
基礎API中的查詢操作在面對大量數據的時候是非常蒼白的,這里Hbase提供了高級的查詢方法:Filter。Filter可以根據簇、列、版本等更多的條件來對數據進行過濾,基於Hbase本身提供的三維有序(主鍵有序、列有序、版本有序),這些Filter可以高效的完成查詢過濾的任務。帶有Filter條件的RPC查詢請求會把Filter分發到各個RegionServer,是一個服務器端(Server-side)的過濾器,這樣也可以降低網絡傳輸的壓力。
要完成一個過濾的操作,至少需要兩個參數。一個是抽象的操作符,Hbase提供了枚舉類型的變量來表示這些抽象的操作符:LESS/LESS_OR_EQUAL/EQUAL/NOT_EUQAL等;另外一個就是具體的比較器(Comparator),代表具體的比較邏輯,如果可以提高字節級的比較、字符串級的比較等。有了這兩個參數,我們就可以清晰的定義篩選的條件,過濾數據。
1、抽象操作符(比較運算符)
LESS <
LESS_OR_EQUAL <=
EQUAL =
NOT_EQUAL <>
GREATER_OR_EQUAL >=
GREATER >
NO_OP 排除所有
2、比較器(指定比較機制)
BinaryComparator 按字節索引順序比較指定字節數組,采用 Bytes.compareTo(byte[])
BinaryPrefixComparator 跟前面相同,只是比較左端的數據是否相同
NullComparator 判斷給定的是否為空
BitComparator 按位比較
RegexStringComparator 提供一個正則的比較器,僅支持 EQUAL 和非 EQUAL
SubstringComparator 判斷提供的子串是否出現在 value 中
二、HBase過濾器的分類
1、比較過濾器
1.行鍵過濾器 RowFilter
public class HbaseFilterTest { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); Filter rowFilter = new RowFilter(CompareOp.GREATER, new BinaryComparator("95007".getBytes())); scan.setFilter(rowFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(cell); } } }
運行結果部分截圖
2.列簇過濾器 FamilyFilter
public class HbaseFilterTest { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); Filter familyFilter = new FamilyFilter(CompareOp.EQUAL, new BinaryComparator("info".getBytes())); scan.setFilter(familyFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(cell); } } } }
運行結果部分截圖
3.列過濾器 QualifierFilter
public class HbaseFilterTest { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); Filter qualifierFilter = new QualifierFilter(CompareOp.EQUAL, new BinaryComparator("name".getBytes())); scan.setFilter(qualifierFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(cell); } } } }
運行結果部分截圖
4.值過濾器 ValueFilter
public class HbaseFilterTest { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); Filter valueFilter = new ValueFilter(CompareOp.EQUAL, new SubstringComparator("男")); scan.setFilter(valueFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(cell); } } } }
運行結果部分截圖
5.時間戳過濾器 TimestampsFilter
public class HbaseFilterTest { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); List<Long> list = new ArrayList<>(); list.add(1522469029503l); TimestampsFilter timestampsFilter = new TimestampsFilter(list); scan.setFilter(timestampsFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(Bytes.toString(cell.getRow()) + "\t" + Bytes.toString(cell.getFamily()) + "\t" + Bytes.toString(cell.getQualifier()) + "\t" + Bytes.toString(cell.getValue()) + "\t" + cell.getTimestamp()); } } } }
運行結果部分截圖
2、專用過濾器
1.單列值過濾器 SingleColumnValueFilter
會返回滿足條件的整行
public class HbaseFilterTest2 { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); SingleColumnValueFilter singleColumnValueFilter = new SingleColumnValueFilter( "info".getBytes(), "name".getBytes(), CompareOp.EQUAL, new SubstringComparator("劉晨")); //如果不設置為 true,則那些不包含指定 column 的行也會返回 singleColumnValueFilter.setFilterIfMissing(true); scan.setFilter(singleColumnValueFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(Bytes.toString(cell.getRow()) + "\t" + Bytes.toString(cell.getFamily()) + "\t" + Bytes.toString(cell.getQualifier()) + "\t" + Bytes.toString(cell.getValue()) + "\t" + cell.getTimestamp()); } } } }
運行結果部分截圖
2.單列值排除器 SingleColumnValueExcludeFilter
public class HbaseFilterTest2 { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); SingleColumnValueExcludeFilter singleColumnValueExcludeFilter = new SingleColumnValueExcludeFilter( "info".getBytes(), "name".getBytes(), CompareOp.EQUAL, new SubstringComparator("劉晨")); singleColumnValueExcludeFilter.setFilterIfMissing(true); scan.setFilter(singleColumnValueExcludeFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(Bytes.toString(cell.getRow()) + "\t" + Bytes.toString(cell.getFamily()) + "\t" + Bytes.toString(cell.getQualifier()) + "\t" + Bytes.toString(cell.getValue()) + "\t" + cell.getTimestamp()); } } } }
運行結果部分截圖
3.前綴過濾器 PrefixFilter----針對行鍵
public class HbaseFilterTest2 { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); PrefixFilter prefixFilter = new PrefixFilter("9501".getBytes()); scan.setFilter(prefixFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(Bytes.toString(cell.getRow()) + "\t" + Bytes.toString(cell.getFamily()) + "\t" + Bytes.toString(cell.getQualifier()) + "\t" + Bytes.toString(cell.getValue()) + "\t" + cell.getTimestamp()); } } } }
運行結果部分截圖
4.列前綴過濾器 ColumnPrefixFilter
public class HbaseFilterTest2 { private static final String ZK_CONNECT_KEY = "hbase.zookeeper.quorum"; private static final String ZK_CONNECT_VALUE = "node21:2181,node22:2181,node23:2181"; private static Connection conn = null; private static Admin admin = null; public static void main(String[] args) throws Exception { Configuration conf = HBaseConfiguration.create(); conf.set(ZK_CONNECT_KEY, ZK_CONNECT_VALUE); conn = ConnectionFactory.createConnection(conf); admin = conn.getAdmin(); Table table = conn.getTable(TableName.valueOf("student")); Scan scan = new Scan(); ColumnPrefixFilter columnPrefixFilter = new ColumnPrefixFilter("name".getBytes()); scan.setFilter(columnPrefixFilter); ResultScanner resultScanner = table.getScanner(scan); for(Result result : resultScanner) { List<Cell> cells = result.listCells(); for(Cell cell : cells) { System.out.println(Bytes.toString(cell.getRow()) + "\t" + Bytes.toString(cell.getFamily()) + "\t" + Bytes.toString(cell.getQualifier()) + "\t" + Bytes.toString(cell.getValue()) + "\t" + cell.getTimestamp()); } } } }
運行結果部分截圖