Review Board 1.7.22


ProgressReporter should work with both old and new MR API

Review Request #4971 - Created May 2, 2012 and updated

Travis Crawford
HCATALOG-373
Reviewers
hcatalog
francisliu
hcatalog-git
Update ProgressReporter to work with both old and new mapreduce API. Delay creating the base record reader so we have a StatusReporter and can use counters.
"ant clean test" passes

I can run pig+hcatalog queries using Elephant-Bird deprecated API wrappers, which is why this issue originally came up.
src/java/org/apache/hcatalog/common/HCatUtil.java
Revision cb6404a New Change
[20] 46 lines
[+20]
47
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
47
import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
48
import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
48
import org.apache.hadoop.hive.thrift.DelegationTokenIdentifier;
49
import org.apache.hadoop.io.Text;
49
import org.apache.hadoop.io.Text;
50
import org.apache.hadoop.mapred.JobClient;
50
import org.apache.hadoop.mapred.JobClient;
51
import org.apache.hadoop.mapred.JobConf;
51
import org.apache.hadoop.mapred.JobConf;

    
   
52
import org.apache.hadoop.mapreduce.InputSplit;
52
import org.apache.hadoop.mapreduce.JobContext;
53
import org.apache.hadoop.mapreduce.JobContext;
53
import org.apache.hadoop.security.token.Token;
54
import org.apache.hadoop.security.token.Token;
54
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
55
import org.apache.hadoop.security.token.delegation.AbstractDelegationTokenIdentifier;
55
import org.apache.hadoop.util.ReflectionUtils;
56
import org.apache.hadoop.util.ReflectionUtils;
56
import org.apache.hcatalog.data.Pair;
57
import org.apache.hcatalog.data.Pair;
57
import org.apache.hcatalog.data.schema.HCatFieldSchema;
58
import org.apache.hcatalog.data.schema.HCatFieldSchema;
58
import org.apache.hcatalog.data.schema.HCatSchema;
59
import org.apache.hcatalog.data.schema.HCatSchema;
59
import org.apache.hcatalog.data.schema.HCatSchemaUtils;
60
import org.apache.hcatalog.data.schema.HCatSchemaUtils;
60
import org.apache.hcatalog.mapreduce.FosterStorageHandler;
61
import org.apache.hcatalog.mapreduce.FosterStorageHandler;
61
import org.apache.hcatalog.mapreduce.HCatOutputFormat;
62
import org.apache.hcatalog.mapreduce.HCatOutputFormat;

    
   
63
import org.apache.hcatalog.mapreduce.HCatSplit;
62
import org.apache.hcatalog.mapreduce.HCatStorageHandler;
64
import org.apache.hcatalog.mapreduce.HCatStorageHandler;
63
import org.apache.hcatalog.mapreduce.InputJobInfo;
65
import org.apache.hcatalog.mapreduce.InputJobInfo;
64
import org.apache.hcatalog.mapreduce.OutputJobInfo;
66
import org.apache.hcatalog.mapreduce.OutputJobInfo;
65
import org.apache.hcatalog.mapreduce.PartInfo;
67
import org.apache.hcatalog.mapreduce.PartInfo;
66
import org.apache.hcatalog.mapreduce.StorerInfo;
68
import org.apache.hcatalog.mapreduce.StorerInfo;
[+20] [20] 523 lines
[+20] [+] public static void copyJobPropertiesToJobConf(
590
    {
592
    {
591
      for (Map.Entry<String, String> entry : jobProperties.entrySet()) {
593
      for (Map.Entry<String, String> entry : jobProperties.entrySet()) {
592
        jobConf.set(entry.getKey(), entry.getValue());
594
        jobConf.set(entry.getKey(), entry.getValue());
593
      }
595
      }
594
    }
596
    }

    
   
597

   

    
   
598
  /**

    
   
599
   * Casts an InputSplit into a HCatSplit, providing a useful error message if the cast fails.

    
   
600
   * @param split the InputSplit

    
   
601
   * @return the HCatSplit

    
   
602
   * @throws IOException

    
   
603
   */

    
   
604
  public static HCatSplit castToHCatSplit(InputSplit split) throws IOException {

    
   
605
    if (split instanceof HCatSplit) {

    
   
606
      return (HCatSplit) split;

    
   
607
    } else {

    
   
608
      throw new IOException("Split must be " + HCatSplit.class.getName()

    
   
609
          + " but found " + split.getClass().getName());

    
   
610
    }

    
   
611
  }

    
   
612

   
595
}
613
}
src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java
Revision 268167e New Change
 
src/java/org/apache/hcatalog/mapreduce/HCatRecordReader.java
Revision 65f96f4 New Change
 
src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java
Revision fb379cd New Change
 
  1. src/java/org/apache/hcatalog/common/HCatUtil.java: Loading...
  2. src/java/org/apache/hcatalog/mapreduce/HCatBaseInputFormat.java: Loading...
  3. src/java/org/apache/hcatalog/mapreduce/HCatRecordReader.java: Loading...
  4. src/java/org/apache/hcatalog/mapreduce/ProgressReporter.java: Loading...