Review Board 1.7.22


SQOOP-1190 Class HCatHadoopShims will be removed in HCatalog 0.12

Review Request #13963 - Created Sept. 4, 2013 and updated

Jarek Cecho
SQOOP-1190
Reviewers
Sqoop
sqoop-trunk
I've used reflection to find out what Shim layer is available.
Tested both with HCatalog 0.5 and current trunk (future 0.12).

Diff revision 1 (Latest)

  1. src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java: Loading...
src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java
Revision a109b4055633415dbfced23a1cb4109dc2575a05 New Change
[20] 56 lines
[+20]
57
import org.apache.hcatalog.data.schema.HCatFieldSchema;
57
import org.apache.hcatalog.data.schema.HCatFieldSchema;
58
import org.apache.hcatalog.data.schema.HCatSchema;
58
import org.apache.hcatalog.data.schema.HCatSchema;
59
import org.apache.hcatalog.mapreduce.HCatInputFormat;
59
import org.apache.hcatalog.mapreduce.HCatInputFormat;
60
import org.apache.hcatalog.mapreduce.HCatOutputFormat;
60
import org.apache.hcatalog.mapreduce.HCatOutputFormat;
61
import org.apache.hcatalog.mapreduce.OutputJobInfo;
61
import org.apache.hcatalog.mapreduce.OutputJobInfo;
62
import org.apache.hcatalog.shims.HCatHadoopShims;

   
63
import org.apache.sqoop.config.ConfigurationConstants;
62
import org.apache.sqoop.config.ConfigurationConstants;
64
import org.apache.sqoop.hive.HiveTypes;
63
import org.apache.sqoop.hive.HiveTypes;
65
import org.apache.sqoop.manager.ConnManager;
64
import org.apache.sqoop.manager.ConnManager;
66
import org.apache.sqoop.util.Executor;
65
import org.apache.sqoop.util.Executor;
67
import org.apache.sqoop.util.LoggingAsyncSink;
66
import org.apache.sqoop.util.LoggingAsyncSink;
[+20] [20] 781 lines
[+20] [+] public static boolean isLocalJobTracker(Job job) {
849
      || (jtAddr2 != null && jtAddr2.equals("local"));
848
      || (jtAddr2 != null && jtAddr2.equals("local"));
850
  }
849
  }
851

    
   
850

   
852
  public void invokeOutputCommitterForLocalMode(Job job) throws IOException {
851
  public void invokeOutputCommitterForLocalMode(Job job) throws IOException {
853
    if (isLocalJobTracker(job) && isHadoop1()) {
852
    if (isLocalJobTracker(job) && isHadoop1()) {

    
   
853
      // HCatalog 0.11- do have special class HCatHadoopShims, however this

    
   
854
      // class got merged into Hive Shim layer in 0.12+. Following method will

    
   
855
      // try to find correct implementation via reflection.

    
   
856

   

    
   
857
      // Final Shim layer

    
   
858
      Object shimLayer = null;

    
   
859
      Class shimClass = null;

    
   
860

   

    
   
861
      // Let's try Hive 0.11-

    
   
862
      try {

    
   
863
        shimClass = Class.forName("org.apache.hcatalog.shims.HCatHadoopShims");

    
   
864

   

    
   
865
        Class shimInstanceClass = Class.forName("org.apache.hcatalog.shims.HCatHadoopShims$Instance");

    
   
866
        Method getMethod = shimInstanceClass.getMethod("get");

    
   
867

   

    
   
868
        shimLayer = getMethod.invoke(null);

    
   
869
      } catch (Exception e) {

    
   
870
        LOG.debug("Not found HCatalog 0.11- implementation of the Shim layer", e);

    
   
871
      }

    
   
872

   

    
   
873
      // For Hive 0.12+

    
   
874
      if (shimClass == null || shimLayer == null) {

    
   
875
        try {

    
   
876
          shimClass = Class.forName("org.apache.hadoop.hive.shims.HadoopShims$HCatHadoopShims");

    
   
877

   

    
   
878
          Class shimLoader = Class.forName("org.apache.hadoop.hive.shims.ShimLoader");

    
   
879
          Method getHadoopShims = shimLoader.getMethod("getHadoopShims");

    
   
880

   

    
   
881
          Object hadoopShims = getHadoopShims.invoke(null);

    
   
882

   

    
   
883
          Class hadoopShimClass = Class.forName("org.apache.hadoop.hive.shims.HadoopShims");

    
   
884
          Method getHCatShim = hadoopShimClass.getMethod("getHCatShim");

    
   
885

   

    
   
886
          shimLayer = getHCatShim.invoke(hadoopShims);

    
   
887
        } catch (Exception e) {

    
   
888
          LOG.debug("Not found HCatalog 0.12+ implementation of the Shim layer", e);

    
   
889
        }

    
   
890
      }

    
   
891

   

    
   
892
      if (shimClass == null || shimLayer == null) {

    
   
893
        throw new IOException("Did not found HCatalog shim layer to commit the job");

    
   
894
      }

    
   
895

   

    
   
896
      // Part that is the same for both shim layer implementations

    
   
897
      try {

    
   
898
        Method commitJobMethod = shimClass.getMethod("commitJob", OutputFormat.class, Job.class);
854
      LOG.info("Explicitly committing job in local mode");
899
        LOG.info("Explicitly committing job in local mode");
855
      HCatHadoopShims.Instance.get().commitJob(new HCatOutputFormat(), job);
900
        commitJobMethod.invoke(shimLayer, new HCatOutputFormat(), job);

    
   
901
      } catch (Exception e) {

    
   
902
        throw new RuntimeException("Can't explicitly commit job", e);

    
   
903
      }
856
    }
904
    }
857
  }
905
  }
858

    
   
906

   
859
  public void launchHCatCli(String cmdLine)
907
  public void launchHCatCli(String cmdLine)
860
    throws IOException {
908
    throws IOException {
[+20] [20] 355 lines
  1. src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatUtilities.java: Loading...