Review Board 1.7.22


HIVE-1644 Use filter pushdown for automatically accessing indexes

Review Request #558 - Created April 7, 2011 and updated

Russell Melick
trunk
HIVE-1644
Reviewers
hive
hive-git
Review request for HIVE-1644.12.patch

 

Diff revision 4

This is not the most recent revision of the diff. The latest diff is revision 7. See what's changed.

1 2 3 4 5 6 7
1 2 3 4 5 6 7

  1. common/src/java/org/apache/hadoop/hive/conf/HiveConf.java: Loading...
  2. conf/hive-default.xml: Loading...
  3. ql/src/java/org/apache/hadoop/hive/ql/Driver.java: Loading...
  4. ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java: Loading...
  5. ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java: Loading...
  6. ql/src/java/org/apache/hadoop/hive/ql/index/AbstractIndexHandler.java: Loading...
  7. ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java: Loading...
  8. ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexQueryContext.java: Loading...
  9. ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java: Loading...
  10. ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java: Loading...
  11. ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java: Loading...
  12. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java: Loading...
  13. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java: Loading...
  14. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java: Loading...
  15. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java: Loading...
  16. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java: Loading...
  17. ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java: Loading...
  18. ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java: Loading...
  19. ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java: Loading...
  20. ql/src/test/queries/clientpositive/index_auto.q: Loading...
  21. ql/src/test/queries/clientpositive/index_auto_file_format.q: Loading...
  22. ql/src/test/queries/clientpositive/index_auto_multiple.q: Loading...
  23. ql/src/test/queries/clientpositive/index_auto_partitioned.q: Loading...
  24. ql/src/test/queries/clientpositive/index_auto_unused.q: Loading...
  25. ql/src/test/results/clientpositive/index_auto.q.out: Loading...
  26. ql/src/test/results/clientpositive/index_auto_file_format.q.out: Loading...
  27. ql/src/test/results/clientpositive/index_auto_multiple.q.out: Loading...
  28. ql/src/test/results/clientpositive/index_auto_partitioned.q.out: Loading...
  29. ql/src/test/results/clientpositive/index_auto_unused.q.out: Loading...
common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Revision 2cdaeb6 New Change
[20] 351 lines
[+20] [+] public class HiveConf extends Configuration {
352

    
   
352

   
353
    HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false),
353
    HIVE_COMBINE_INPUT_FORMAT_SUPPORTS_SPLITTABLE("hive.hadoop.supports.splittable.combineinputformat", false),
354

    
   
354

   
355
    // Optimizer
355
    // Optimizer
356
    HIVEOPTCP("hive.optimize.cp", true), // column pruner
356
    HIVEOPTCP("hive.optimize.cp", true), // column pruner

    
   
357
    HIVEOPTINDEXFILTER("hive.optimize.index.filter", false), // automatically use indexes
357
    HIVEOPTPPD("hive.optimize.ppd", true), // predicate pushdown
358
    HIVEOPTPPD("hive.optimize.ppd", true), // predicate pushdown
358
    // push predicates down to storage handlers
359
    // push predicates down to storage handlers
359
    HIVEOPTPPD_STORAGE("hive.optimize.ppd.storage", true),
360
    HIVEOPTPPD_STORAGE("hive.optimize.ppd.storage", true),
360
    HIVEOPTGROUPBY("hive.optimize.groupby", true), // optimize group by
361
    HIVEOPTGROUPBY("hive.optimize.groupby", true), // optimize group by
361
    HIVEOPTBUCKETMAPJOIN("hive.optimize.bucketmapjoin", false), // optimize bucket map join
362
    HIVEOPTBUCKETMAPJOIN("hive.optimize.bucketmapjoin", false), // optimize bucket map join
362
    HIVEOPTSORTMERGEBUCKETMAPJOIN("hive.optimize.bucketmapjoin.sortedmerge", false), // try to use sorted merge bucket map join
363
    HIVEOPTSORTMERGEBUCKETMAPJOIN("hive.optimize.bucketmapjoin.sortedmerge", false), // try to use sorted merge bucket map join
363
    HIVEOPTREDUCEDEDUPLICATION("hive.optimize.reducededuplication", true),
364
    HIVEOPTREDUCEDEDUPLICATION("hive.optimize.reducededuplication", true),
364

    
   
365

   

    
   
366
    // Indexes

    
   
367
    HIVEOPTINDEXFILTER_COMPACT_MINSIZE("hive.optimize.index.filter.compact.minsize", (long) 5 * 1024 * 1024 * 1024), // 5G

    
   
368
    HIVEOPTINDEXFILTER_COMPACT_MAXSIZE("hive.optimize.index.filter.compact.maxsize", (long) -1), // infinity

    
   
369

   
365
    // Statistics
370
    // Statistics
366
    HIVESTATSAUTOGATHER("hive.stats.autogather", true),
371
    HIVESTATSAUTOGATHER("hive.stats.autogather", true),
367
    HIVESTATSDBCLASS("hive.stats.dbclass",
372
    HIVESTATSDBCLASS("hive.stats.dbclass",
368
        "jdbc:derby"), // other options are jdbc:mysql and hbase as defined in StatsSetupConst.java
373
        "jdbc:derby"), // other options are jdbc:mysql and hbase as defined in StatsSetupConst.java
369
    HIVESTATSJDBCDRIVER("hive.stats.jdbcdriver",
374
    HIVESTATSJDBCDRIVER("hive.stats.jdbcdriver",
[+20] [20] 370 lines
conf/hive-default.xml
Revision 79ea477 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/Driver.java
Revision ca337a8 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java
Revision 69ee03b New Change
 
ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
Revision c02d90b New Change
 
ql/src/java/org/apache/hadoop/hive/ql/index/AbstractIndexHandler.java
Revision dd0186d New Change
 
ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java
Revision 411b78f New Change
 
ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexQueryContext.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java
Revision 1f01446 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
Revision 50db44c New Change
 
ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java
Revision 6162676 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java
Revision 0ae9fa2 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java
Revision 374e123 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Revision c41bb32 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java
Revision 73391e9 New Change
 
ql/src/test/queries/clientpositive/index_auto.q
New File
 
ql/src/test/queries/clientpositive/index_auto_file_format.q
New File
 
ql/src/test/queries/clientpositive/index_auto_multiple.q
New File
 
ql/src/test/queries/clientpositive/index_auto_partitioned.q
New File
 
ql/src/test/queries/clientpositive/index_auto_unused.q
New File
 
ql/src/test/results/clientpositive/index_auto.q.out
New File
 
ql/src/test/results/clientpositive/index_auto_file_format.q.out
New File
 
ql/src/test/results/clientpositive/index_auto_multiple.q.out
New File
 
ql/src/test/results/clientpositive/index_auto_partitioned.q.out
New File
 
ql/src/test/results/clientpositive/index_auto_unused.q.out
New File
 
  1. common/src/java/org/apache/hadoop/hive/conf/HiveConf.java: Loading...
  2. conf/hive-default.xml: Loading...
  3. ql/src/java/org/apache/hadoop/hive/ql/Driver.java: Loading...
  4. ql/src/java/org/apache/hadoop/hive/ql/exec/ExecDriver.java: Loading...
  5. ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java: Loading...
  6. ql/src/java/org/apache/hadoop/hive/ql/index/AbstractIndexHandler.java: Loading...
  7. ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexHandler.java: Loading...
  8. ql/src/java/org/apache/hadoop/hive/ql/index/HiveIndexQueryContext.java: Loading...
  9. ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java: Loading...
  10. ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java: Loading...
  11. ql/src/java/org/apache/hadoop/hive/ql/optimizer/GenMRTableScan1.java: Loading...
  12. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/IndexWhereResolver.java: Loading...
  13. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/PhysicalOptimizer.java: Loading...
  14. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcCtx.java: Loading...
  15. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereProcessor.java: Loading...
  16. ql/src/java/org/apache/hadoop/hive/ql/optimizer/physical/index/IndexWhereTaskDispatcher.java: Loading...
  17. ql/src/java/org/apache/hadoop/hive/ql/parse/ParseContext.java: Loading...
  18. ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java: Loading...
  19. ql/src/java/org/apache/hadoop/hive/ql/plan/MapredWork.java: Loading...
  20. ql/src/test/queries/clientpositive/index_auto.q: Loading...
  21. ql/src/test/queries/clientpositive/index_auto_file_format.q: Loading...
  22. ql/src/test/queries/clientpositive/index_auto_multiple.q: Loading...
  23. ql/src/test/queries/clientpositive/index_auto_partitioned.q: Loading...
  24. ql/src/test/queries/clientpositive/index_auto_unused.q: Loading...
  25. ql/src/test/results/clientpositive/index_auto.q.out: Loading...
  26. ql/src/test/results/clientpositive/index_auto_file_format.q.out: Loading...
  27. ql/src/test/results/clientpositive/index_auto_multiple.q.out: Loading...
  28. ql/src/test/results/clientpositive/index_auto_partitioned.q.out: Loading...
  29. ql/src/test/results/clientpositive/index_auto_unused.q.out: Loading...