SmallFileMerging

--Create table testtable
--(hash string,
--timstim timestamp,
--payload string)
--Partitioned By (string year, string month, string day);

--https://issues.apache.org/jira/browse/HIVE-3509
--https://issues.apache.org/jira/browse/HIVE-15774  hive.txn.strict.locking.mode

SET hive.execution.engine=mr;
SET hive.support.quoted.identifiers=NONE;
SET mapreduce.INPUT.fileinputformat.split.maxsize=500000000; --optimized for mapper buffers
SET mapreduce.INPUT.fileinputformat.split.minsize=500000000;
SET mapreduce.map.output.compress=TRUE; --improve shuffle transfer
SET mapreduce.map.output.compress.codec=org.apache.hadoop.io.compress.SnappyCodec;
SET mapreduce.reduce.memory.mb=1600; --increase if you get OOM or Heap Errors
SET mapreduce.reduce.java.opts=-Xmx14g; --increase if you get OOM or Heap Errors
SET hive.EXEC.reducers.MAX=1; --control max output files into the partition or table

INSERT OVERWRITE TABLE testtable PARTITION (YEAR=‘2017’, MONTH=‘05’, DAY=‘22’)  --Specific Partition Prevents Locking OthersSELECT `(year|month|day)?+.+`
FROM testtable
WHERE YEAR=‘2017 AND MONTH=‘05’ AND DAY=‘22’ --Specific WHERE Partition Prevents Locking Others
SORT BY hash, timstim;  --optimize your ORC files for your query predicate patterns