Review Board 1.7.22


PIG-3502 Initial implementation of TezLauncher

Review Request #14601 - Created Oct. 11, 2013 and submitted

Cheolsoo Park
tez-branch
PIG-3502
Reviewers
pig
daijy, mwagner, rohini
pig-git
Initial implementation of TezLauncher. The patch includes the following changes:
- Refactors Launcher and MapReduceLauncher. Basically, I moved what's MR-specific to MapReduceLauncher and kept what's common in Launcher.
- Removes kill() method from ExecutionEngine interface. It was used merely by Launcher, and it didn't seem useful. 
- Implements the DAG submission of TezLauncher. Basically, I overrode the submit() method in TezJob (extension of ControlledJob) that calls DAGClient.submit().
- Adds a skeleton TezStats class that extends PigStats. This class needs to be implemented to monitor job status after submission.
- Adds a unit test called TestTezLauncher. For now, all it does is to compile queries and submit a DAG. More meaningful unit tests will be added after TezStats is implemented.
- Adds a TezLocalExecType. To start up a YARN mini cluster, I needed non-local Tez mode. So I created "tez_local" and "tez" modes.
- Whitespace and indentation fixes.
TestTezLauncher is added.
build.xml
Revision 0439082 New Change
[20] 344 lines
[+20]
345
            <include name="jackson-core-asl-${jackson.version}.jar"/>
345
            <include name="jackson-core-asl-${jackson.version}.jar"/>
346
            <include name="joda-time-${joda-time.version}.jar"/>
346
            <include name="joda-time-${joda-time.version}.jar"/>
347
            <include name="guava-${guava.version}.jar"/>
347
            <include name="guava-${guava.version}.jar"/>
348
            <include name="automaton-${automaton.version}.jar"/>
348
            <include name="automaton-${automaton.version}.jar"/>
349
            <include name="jansi-${jansi.version}.jar"/>
349
            <include name="jansi-${jansi.version}.jar"/>
350
            <include name="avro-${avro.version}.jar"/>     
350
            <include name="avro-${avro.version}.jar"/>
351
            <include name="avro-mapred-${avro.version}.jar"/>     
351
            <include name="avro-mapred-${avro.version}.jar"/>
352
            <include name="trevni-core-${avro.version}.jar"/>     
352
            <include name="trevni-core-${avro.version}.jar"/>
353
            <include name="trevni-avro-${avro.version}.jar"/>
353
            <include name="trevni-avro-${avro.version}.jar"/>
354
            <include name="snappy-java-${snappy.version}.jar"/>
354
            <include name="snappy-java-${snappy.version}.jar"/>
355
            <include name="asm*.jar"/>
355
            <include name="asm*.jar"/>
356
        </patternset>
356
        </patternset>
357
    </fileset>
357
    </fileset>
[+20] [20] 10 lines
[+20]
368
            <include name="commons*.jar"/>
368
            <include name="commons*.jar"/>
369
            <include name="log4j*.jar"/>
369
            <include name="log4j*.jar"/>
370
            <include name="slf4j*.jar"/>
370
            <include name="slf4j*.jar"/>
371
            <include name="jsp-api*.jar"/>
371
            <include name="jsp-api*.jar"/>
372
            <include name="avro-${avro.version}.jar"/>
372
            <include name="avro-${avro.version}.jar"/>
373
            <include name="avro-mapred-${avro.version}.jar"/>    
373
            <include name="avro-mapred-${avro.version}.jar"/>
374
            <include name="trevni-core-${avro.version}.jar"/>    
374
            <include name="trevni-core-${avro.version}.jar"/>
375
            <include name="trevni-avro-${avro.version}.jar"/>
375
            <include name="trevni-avro-${avro.version}.jar"/>
376
            <include name="snappy-java-${snappy.version}.jar"/>
376
            <include name="snappy-java-${snappy.version}.jar"/>
377
            <include name="asm*.jar"/>
377
            <include name="asm*.jar"/>

    
   
378
            <include name="tez-*-${tez.version}.jar" if="isHadoop23"/>
378
        </patternset>
379
        </patternset>
379
    </fileset>
380
    </fileset>
380

    
   
381

   
381
    <fileset dir="${ivy.lib.dir}" id="core.dependencies.jar">
382
    <fileset dir="${ivy.lib.dir}" id="core.dependencies.jar">
382
        <exclude name="**.*jar"/>
383
        <exclude name="**.*jar"/>
[+20] [20] 168 lines
[+20]
551
    <target name="compile-test" depends="compile, ivy-test">
552
    <target name="compile-test" depends="compile, ivy-test">
552
        <echo>*** Building Test Sources ***</echo>
553
        <echo>*** Building Test Sources ***</echo>
553
        <echo>*** To compile with all warnings enabled, supply -Dall.warnings=1 on command line ***</echo>
554
        <echo>*** To compile with all warnings enabled, supply -Dall.warnings=1 on command line ***</echo>
554
        <echo>*** Else, you will only be warned about deprecations ***</echo>
555
        <echo>*** Else, you will only be warned about deprecations ***</echo>
555
        <compileSources sources="${test.src.dir};${src.shims.test.dir}"
556
        <compileSources sources="${test.src.dir};${src.shims.test.dir}"
556
            excludes="**/PigTestLoader.java **/resources/** perf/**" dist="${test.build.classes}" cp="test.classpath" warnings="${javac.args.warnings}" />
557
            excludes="**/PigTestLoader.java **/resources/** perf/** ${src.exclude.dir}"

    
   
558
            dist="${test.build.classes}" cp="test.classpath" warnings="${javac.args.warnings}" />
557
        <copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>
559
        <copy file="${basedir}/test/hbase-site.xml" tofile="${test.build.classes}/hbase-site.xml"/>
558
        <ivy:cachepath pathid="mr-apps-test-ivy.classpath" conf="test" />
560
        <ivy:cachepath pathid="mr-apps-test-ivy.classpath" conf="test" />
559
        <path id="mr-apps-test.classpath">
561
        <path id="mr-apps-test.classpath">
560
            <fileset dir="${clover.home}" erroronmissingdir="false">
562
            <fileset dir="${clover.home}" erroronmissingdir="false">
561
                <include name="lib/clover.jar"/>
563
                <include name="lib/clover.jar"/>
[+20] [20] 1110 lines
src/META-INF/services/org.apache.pig.ExecType
Revision e995cd5 New Change
 
src/org/apache/pig/backend/executionengine/ExecutionEngine.java
Revision 7d480d7 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
Revision b886f9d New Change
 
src/org/apache/pig/backend/hadoop/executionengine/Launcher.java
Revision 77543c2 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Revision f3be738 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java
Revision 5414906 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/tez/TezExecType.java
Revision 8315e76 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java
Revision 04ffba0 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java
Revision b7d4d98 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java
Revision ff9e950 New Change
 
src/org/apache/pig/backend/hadoop/executionengine/tez/TezLocalExecType.java
New File
 
src/org/apache/pig/backend/hadoop/executionengine/tez/TezStats.java
New File
 
src/org/apache/pig/impl/PigContext.java
Revision 225526f New Change
 
src/org/apache/pig/impl/util/Utils.java
Revision 3720c94 New Change
 
test/org/apache/pig/tez/TestTezCompiler.java
Revision b9e6b48 New Change
 
test/org/apache/pig/tez/TestTezJobControlCompiler.java
Revision f14d49d New Change
 
test/org/apache/pig/tez/TestTezLauncher.java
New File
 
  1. build.xml: Loading...
  2. src/META-INF/services/org.apache.pig.ExecType: Loading...
  3. src/org/apache/pig/backend/executionengine/ExecutionEngine.java: Loading...
  4. src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java: Loading...
  5. src/org/apache/pig/backend/hadoop/executionengine/Launcher.java: Loading...
  6. src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java: Loading...
  7. src/org/apache/pig/backend/hadoop/executionengine/tez/TezDagBuilder.java: Loading...
  8. src/org/apache/pig/backend/hadoop/executionengine/tez/TezExecType.java: Loading...
  9. src/org/apache/pig/backend/hadoop/executionengine/tez/TezJob.java: Loading...
  10. src/org/apache/pig/backend/hadoop/executionengine/tez/TezJobControlCompiler.java: Loading...
  11. src/org/apache/pig/backend/hadoop/executionengine/tez/TezLauncher.java: Loading...
  12. src/org/apache/pig/backend/hadoop/executionengine/tez/TezLocalExecType.java: Loading...
  13. src/org/apache/pig/backend/hadoop/executionengine/tez/TezStats.java: Loading...
  14. src/org/apache/pig/impl/PigContext.java: Loading...
  15. src/org/apache/pig/impl/util/Utils.java: Loading...
  16. test/org/apache/pig/tez/TestTezCompiler.java: Loading...
  17. test/org/apache/pig/tez/TestTezJobControlCompiler.java: Loading...
  18. test/org/apache/pig/tez/TestTezLauncher.java: Loading...