Review Board 1.7.22


HIVE-306 Support INSERT INTO

Review Request #926 - Created June 16, 2011 and submitted

Franklin Hu
HIVE-306
Reviewers
hive
hive
New feature to allow INSERT INTO a table or partition

 

Diff revision 4 (Latest)

1 2 3 4
1 2 3 4

  1. trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java: Loading...
  2. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java: Loading...
  3. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g: Loading...
  4. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java: Loading...
  5. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java: Loading...
  6. trunk/ql/src/test/queries/clientnegative/insert_into1.q: Loading...
  7. trunk/ql/src/test/queries/clientnegative/insert_into2.q: Loading...
  8. trunk/ql/src/test/queries/clientnegative/insert_into3.q: Loading...
  9. trunk/ql/src/test/queries/clientnegative/insert_into4.q: Loading...
  10. trunk/ql/src/test/queries/clientpositive/insert_into1.q: Loading...
  11. trunk/ql/src/test/queries/clientpositive/insert_into2.q: Loading...
  12. trunk/ql/src/test/queries/clientpositive/insert_into3.q: Loading...
  13. trunk/ql/src/test/queries/clientpositive/insert_into4.q: Loading...
  14. trunk/ql/src/test/queries/clientpositive/insert_into5.q: Loading...
  15. trunk/ql/src/test/queries/clientpositive/insert_into6.q: Loading...
  16. trunk/ql/src/test/results/clientnegative/insert_into1.q.out: Loading...
  17. trunk/ql/src/test/results/clientnegative/insert_into2.q.out: Loading...
  18. trunk/ql/src/test/results/clientnegative/insert_into3.q.out: Loading...
  19. trunk/ql/src/test/results/clientnegative/insert_into4.q.out: Loading...
  20. trunk/ql/src/test/results/clientpositive/insert_into1.q.out: Loading...
  21. trunk/ql/src/test/results/clientpositive/insert_into2.q.out: Loading...
  22. trunk/ql/src/test/results/clientpositive/insert_into3.q.out: Loading...
  23. trunk/ql/src/test/results/clientpositive/insert_into4.q.out: Loading...
  24. trunk/ql/src/test/results/clientpositive/insert_into5.q.out: Loading...
  25. trunk/ql/src/test/results/clientpositive/insert_into6.q.out: Loading...
  26. trunk/ql/src/test/results/compiler/errors/missing_overwrite.q.out: Loading...
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java
Revision 1145273 New Change
[20] 304 lines
[+20] [+] private int aggregateStats() {
305
        }
305
        }
306
        tblStats.setStat(StatsSetupConst.TOTAL_SIZE, tableSize);
306
        tblStats.setStat(StatsSetupConst.TOTAL_SIZE, tableSize);
307

    
   
307

   
308
        // In case of a non-partitioned table, the key for stats temporary store is "rootDir"
308
        // In case of a non-partitioned table, the key for stats temporary store is "rootDir"
309
        if (statsAggregator != null) {
309
        if (statsAggregator != null) {
310
          String value;
310
          updateStats(collectableStats, tblStats, statsAggregator, parameters, 
311
          for (String statType : collectableStats) {
311
              work.getAggKey(), atomic);
312
            value = statsAggregator.aggregateStats(work.getAggKey(), statType);

   
313
            if (value != null) {

   
314
              tblStats.setStat(statType, Long.parseLong(value));

   
315
            } else {

   
316
              if (atomic) {

   
317
                throw new HiveException("StatsAggregator failed to get statistics.");

   
318
              }

   
319
            }

   
320
          }

   
321
          statsAggregator.cleanUp(work.getAggKey());
312
          statsAggregator.cleanUp(work.getAggKey());
322
        }
313
        }
323
      } else {
314
      } else {
324
        // Partitioned table:
315
        // Partitioned table:
325
        // Need to get the old stats of the partition
316
        // Need to get the old stats of the partition
[+20] [20] 27 lines
[+20] private int aggregateStats() {
353
          String partitionID = work.getAggKey() + Warehouse.makePartPath(partn.getSpec());
344
          String partitionID = work.getAggKey() + Warehouse.makePartPath(partn.getSpec());
354

    
   
345

   
355
          LOG.info("Stats aggregator : " + partitionID);
346
          LOG.info("Stats aggregator : " + partitionID);
356

    
   
347

   
357
          if (statsAggregator != null) {
348
          if (statsAggregator != null) {
358
            String value;
349
            updateStats(collectableStats, newPartStats, statsAggregator, 
359
            for (String statType : collectableStats) {
350
                parameters, partitionID, atomic);
360
              value = statsAggregator.aggregateStats(partitionID, statType);

   
361
              if (value != null) {

   
362
                newPartStats.setStat(statType, Long.parseLong(value));

   
363
              } else {

   
364
                if (atomic) {

   
365
                  throw new HiveException("StatsAggregator failed to get statistics.");

   
366
                }

   
367
              }

   
368
            }

   
369
          } else {
351
          } else {
370
            for (String statType : collectableStats) {
352
            for (String statType : collectableStats) {
371
              newPartStats.setStat(statType, currentValues.get(statType));
353
              newPartStats.setStat(statType, currentValues.get(statType));
372
            }
354
            }
373
          }
355
          }
[+20] [20] 75 lines
[+20] [+] private boolean existStats(Map<String, String> parameters) {
449
        || parameters.containsKey(StatsSetupConst.TOTAL_SIZE)
431
        || parameters.containsKey(StatsSetupConst.TOTAL_SIZE)
450
        || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE)
432
        || parameters.containsKey(StatsSetupConst.RAW_DATA_SIZE)
451
        || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS);
433
        || parameters.containsKey(StatsSetupConst.NUM_PARTITIONS);
452
  }
434
  }
453

    
   
435

   

    
   
436
  private void updateStats(List<String> statsList, PartitionStatistics stats,

    
   
437
      StatsAggregator statsAggregator, Map<String, String> parameters,

    
   
438
      String aggKey, boolean atomic) throws HiveException {

    
   
439

   

    
   
440
    String value;

    
   
441
    Long longValue;

    
   
442
    for (String statType : statsList) {

    
   
443
      value = statsAggregator.aggregateStats(aggKey, statType);

    
   
444
      if (value != null) {

    
   
445
        longValue = Long.parseLong(value);

    
   
446

   

    
   
447
        if (!work.getLoadTableDesc().getReplace()) {

    
   
448
          String originalValue = parameters.get(statType);

    
   
449
          if (originalValue != null) {

    
   
450
            longValue += Long.parseLong(originalValue);

    
   
451
          }

    
   
452
        }

    
   
453
        stats.setStat(statType, longValue);
Moved from 315

    
   
454
      } else {
Moved from 316

    
   
455
        if (atomic) {
Moved from 317

    
   
456
          throw new HiveException("StatsAggregator failed to get statistics.");
Moved from 318

    
   
457
        }
Moved from 319

    
   
458
      }
Moved from 320

    
   
459
    }

    
   
460
  }

    
   
461

   
454
  /**
462
  /**
455
   * Get the list of partitions that need to update statistics.
463
   * Get the list of partitions that need to update statistics.
456
   * TODO: we should reuse the Partitions generated at compile time
464
   * TODO: we should reuse the Partitions generated at compile time
457
   * since getting the list of partitions is quite expensive.
465
   * since getting the list of partitions is quite expensive.
458
   *
466
   *
[+20] [20] 60 lines
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
Revision 1145273 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g
Revision 1145273 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java
Revision 1145273 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Revision 1145273 New Change
 
trunk/ql/src/test/queries/clientnegative/insert_into1.q
New File
 
trunk/ql/src/test/queries/clientnegative/insert_into2.q
New File
 
trunk/ql/src/test/queries/clientnegative/insert_into3.q
New File
 
trunk/ql/src/test/queries/clientnegative/insert_into4.q
New File
 
trunk/ql/src/test/queries/clientpositive/insert_into1.q
New File
 
trunk/ql/src/test/queries/clientpositive/insert_into2.q
New File
 
trunk/ql/src/test/queries/clientpositive/insert_into3.q
New File
 
trunk/ql/src/test/queries/clientpositive/insert_into4.q
New File
 
trunk/ql/src/test/queries/clientpositive/insert_into5.q
New File
 
trunk/ql/src/test/queries/clientpositive/insert_into6.q
New File
 
trunk/ql/src/test/results/clientnegative/insert_into1.q.out
New File
 
trunk/ql/src/test/results/clientnegative/insert_into2.q.out
New File
 
trunk/ql/src/test/results/clientnegative/insert_into3.q.out
New File
 
trunk/ql/src/test/results/clientnegative/insert_into4.q.out
New File
 
trunk/ql/src/test/results/clientpositive/insert_into1.q.out
New File
 
trunk/ql/src/test/results/clientpositive/insert_into2.q.out
New File
 
trunk/ql/src/test/results/clientpositive/insert_into3.q.out
New File
 
trunk/ql/src/test/results/clientpositive/insert_into4.q.out
New File
 
trunk/ql/src/test/results/clientpositive/insert_into5.q.out
New File
 
trunk/ql/src/test/results/clientpositive/insert_into6.q.out
New File
 
trunk/ql/src/test/results/compiler/errors/missing_overwrite.q.out
Revision 1145273 New Change
 
  1. trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/StatsTask.java: Loading...
  2. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java: Loading...
  3. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/Hive.g: Loading...
  4. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/QBParseInfo.java: Loading...
  5. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java: Loading...
  6. trunk/ql/src/test/queries/clientnegative/insert_into1.q: Loading...
  7. trunk/ql/src/test/queries/clientnegative/insert_into2.q: Loading...
  8. trunk/ql/src/test/queries/clientnegative/insert_into3.q: Loading...
  9. trunk/ql/src/test/queries/clientnegative/insert_into4.q: Loading...
  10. trunk/ql/src/test/queries/clientpositive/insert_into1.q: Loading...
  11. trunk/ql/src/test/queries/clientpositive/insert_into2.q: Loading...
  12. trunk/ql/src/test/queries/clientpositive/insert_into3.q: Loading...
  13. trunk/ql/src/test/queries/clientpositive/insert_into4.q: Loading...
  14. trunk/ql/src/test/queries/clientpositive/insert_into5.q: Loading...
  15. trunk/ql/src/test/queries/clientpositive/insert_into6.q: Loading...
  16. trunk/ql/src/test/results/clientnegative/insert_into1.q.out: Loading...
  17. trunk/ql/src/test/results/clientnegative/insert_into2.q.out: Loading...
  18. trunk/ql/src/test/results/clientnegative/insert_into3.q.out: Loading...
  19. trunk/ql/src/test/results/clientnegative/insert_into4.q.out: Loading...
  20. trunk/ql/src/test/results/clientpositive/insert_into1.q.out: Loading...
  21. trunk/ql/src/test/results/clientpositive/insert_into2.q.out: Loading...
  22. trunk/ql/src/test/results/clientpositive/insert_into3.q.out: Loading...
  23. trunk/ql/src/test/results/clientpositive/insert_into4.q.out: Loading...
  24. trunk/ql/src/test/results/clientpositive/insert_into5.q.out: Loading...
  25. trunk/ql/src/test/results/clientpositive/insert_into6.q.out: Loading...
  26. trunk/ql/src/test/results/compiler/errors/missing_overwrite.q.out: Loading...