Hive Bloom Filters

set hive.tez.exec.print.summary=true;
set hive.execution.engine=tez;
create table stats_test_2 (line string) stored as orc;
insert into table stats_test_2 values ('testval1'),('testval2');
insert into table stats_test_2 values ('testval3'),('testval4');
ALTER TABLE stats_test_2 SET TBLPROPERTIES('orc.create.index'='true','orc.bloom.filter.columns'='line','orc.bloom.filter.fpp'='0.05');
Analyze table stats_test_2 COMPUTE STATISTICS FOR COLUMNS;
select count(*), line from stats_test_2 where line='testval1' group by line;


ALTER TABLE stats_test_2 SET TBLPROPERTIES('orc.create.index'='true','orc.bloom.filter.columns'=col,col2','orc.bloom.filter.fpp'='0.05');

--Analyze DOES NOT create bloom filters! They are only created at INSERT
insert overwrite table $name select * from $table sort by line;

Then orcfile dump and look for the bloom filter entry in the stripe –

/usr/hdp/2.5.3.0-37/hive/bin/hive --orcfiledump /apps/hive/warehouse/jniemie8.db/stats_test_2/000000_0
Processing data file /apps/hive/warehouse/jniemie8.db/stats_test_2/000000_0 [length: 406]
Structure for /apps/hive/warehouse/jniemie8.db/stats_test_2/000000_0
File Version: 0.12 with HIVE_13083
2017-07-12 10:41:49,382 INFO  [main] orc.ReaderImpl: Reading ORC rows from /apps/hive/warehouse/jniemie8.db/stats_test_2/000000_0 with {include: null, offset: 0, length: 9223372036854775807}
2017-07-12 10:41:49,384 INFO  [main] orc.RecordReaderImpl: Reader schema not provided -- using file schema struct<_col0:string>
Rows: 2
Compression: ZLIB
Compression size: 262144
Type: struct<_col0:string>

Stripe Statistics:
  Stripe 1:
    Column 0: count: 2 hasNull: false
    Column 1: count: 2 hasNull: false min: testval3 max: testval4 sum: 16

File Statistics:
  Column 0: count: 2 hasNull: false
  Column 1: count: 2 hasNull: false min: testval3 max: testval4 sum: 16

Stripes:
  Stripe: offset: 3 data: 22 rows: 2 tail: 60 index: 185
    Stream: column 0 section ROW_INDEX start: 3 length 11
    Stream: column 1 section ROW_INDEX start: 14 length 35
    Stream: column 1 section BLOOM_FILTER start: 49 length 139
    Stream: column 1 section DATA start: 188 length 16
    Stream: column 1 section LENGTH start: 204 length 6
    Encoding column 0: DIRECT
    Encoding column 1: DIRECT_V2

File length: 406 bytes
Padding length: 0 bytes
Padding ratio: 0%