Review Board 1.7.22


PIG-3531 Add TezMiniCluster for unit tests

Review Request #14801 - Created Oct. 21, 2013 and submitted

Cheolsoo Park
tez-branch
PIG-3531
Reviewers
pig
daijy, mwagner, rohini
pig-git
Add TezMiniCluster that allows us to write unit tests. The TezMiniCluster class extends MiniGenericCluster and set up Tez-specific properties and jars.
Updated TestTezLauncher using TezMiniCluster. Now it asserts the output is the same as what's expected.
build.xml
Revision 1b65894 New Change
[20] 87 lines
[+20]
88
    <!-- Maintain old pig.jar in top level directory. -->
88
    <!-- Maintain old pig.jar in top level directory. -->
89
    <property name="output.jarfile.backcompat" value="${basedir}/${name}.jar" />
89
    <property name="output.jarfile.backcompat" value="${basedir}/${name}.jar" />
90
    <property name="output.jarfile.backcompat.withouthadoop" value="${basedir}/${name}-withouthadoop.jar" />
90
    <property name="output.jarfile.backcompat.withouthadoop" value="${basedir}/${name}-withouthadoop.jar" />
91

    
   
91

   
92
    <!-- test properties -->
92
    <!-- test properties -->

    
   
93
    <condition property="test.exec.type" value="mr">

    
   
94
        <!-- By default, test.exec.type is mapreduce -->

    
   
95
        <not>

    
   
96
            <isset property="${exectype}"/>

    
   
97
        </not>

    
   
98
    </condition>
93
    <property name="test.src.dir" value="${basedir}/test" />
99
    <property name="test.src.dir" value="${basedir}/test" />
94
    <property name="test.build.dir" value="${build.dir}/test" />
100
    <property name="test.build.dir" value="${build.dir}/test" />
95
    <property name="test.build.classes" value="${test.build.dir}/classes" />
101
    <property name="test.build.classes" value="${test.build.dir}/classes" />
96
    <property name="test.log.dir" value="${test.build.dir}/logs" />
102
    <property name="test.log.dir" value="${test.build.dir}/logs" />
97
    <property name="test.timeout" value="7200000" />
103
    <property name="test.timeout" value="7200000" />
98
    <property name="test.junit.output.format" value="plain" />
104
    <property name="test.junit.output.format" value="plain" />
99
    <property name="test.commit.file" value="${test.src.dir}/commit-tests"/>
105
    <property name="test.commit.file" value="${test.src.dir}/commit-tests"/>
100
    <property name="test.unit.file" value="${test.src.dir}/unit-tests"/>
106
    <property name="test.unit.file" value="${test.src.dir}/unit-tests"/>
101
    <property name="test.smoke.file" value="${test.src.dir}/smoke-tests"/>
107
    <property name="test.smoke.file" value="${test.src.dir}/smoke-tests"/>
102
    <property name="test.all.file" value="${test.src.dir}/all-tests"/>
108
    <property name="test.all.file" value="${test.src.dir}/all-tests"/>

    
   
109
    <property name="test.tez.file" value="${test.src.dir}/tez-tests"/>
103
    <property name="test.exclude.file" value="${test.src.dir}/excluded-tests"/>
110
    <property name="test.exclude.file" value="${test.src.dir}/excluded-tests"/>
104
    <property name="test.exclude.file.23" value="${test.src.dir}/excluded-tests-23"/>
111
    <property name="test.exclude.file.23" value="${test.src.dir}/excluded-tests-23"/>
105
    <property name="pigunit.jarfile" value="pigunit.jar" />
112
    <property name="pigunit.jarfile" value="pigunit.jar" />
106
    <property name="piggybank.jarfile" value="${basedir}/contrib/piggybank/java/piggybank.jar" />
113
    <property name="piggybank.jarfile" value="${basedir}/contrib/piggybank/java/piggybank.jar" />
107
    <property name="smoke.tests.jarfile" value="${build.dir}/${final.name}-smoketests.jar" />
114
    <property name="smoke.tests.jarfile" value="${build.dir}/${final.name}-smoketests.jar" />
[+20] [20] 723 lines
[+20]
831

    
   
838

   
832
    <target name="test-smoke" depends="compile-test,jar-withouthadoop,debugger.check" description="Run 30 min smoke tests">
839
    <target name="test-smoke" depends="compile-test,jar-withouthadoop,debugger.check" description="Run 30 min smoke tests">
833
        <macro-test-runner test.file="${test.smoke.file}" />
840
        <macro-test-runner test.file="${test.smoke.file}" />
834
    </target>
841
    </target>
835

    
   
842

   

    
   
843
    <target name="test-tez" depends="compile-test,jar-withouthadoop,debugger.check" description="Run tez unit tests">

    
   
844
        <property name="test.exec.type" value="tez" />

    
   
845
        <macro-test-runner test.file="${test.tez.file}" />

    
   
846
    </target>

    
   
847

   
836
    <target name="debugger.check" depends="debugger.set,debugger.unset"/>
848
    <target name="debugger.check" depends="debugger.set,debugger.unset"/>
837
    <target name="debugger.set" if="debugPort">
849
    <target name="debugger.set" if="debugPort">
838
        <property name="debugArgs" value="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=${debugPort}"/>
850
        <property name="debugArgs" value="-Xdebug -Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=${debugPort}"/>
839
        <echo>debugArgs: ${debugArgs}</echo>
851
        <echo>debugArgs: ${debugArgs}</echo>
840
    </target>
852
    </target>
[+20] [20] 7 lines
[+20]
848
        <delete dir="${test.log.dir}"/>
860
        <delete dir="${test.log.dir}"/>
849
        <mkdir dir="${test.log.dir}"/>
861
        <mkdir dir="${test.log.dir}"/>
850
        <tempfile property="junit.tmp.dir" prefix="pig_junit_tmp" destDir="${java.io.tmpdir}" />
862
        <tempfile property="junit.tmp.dir" prefix="pig_junit_tmp" destDir="${java.io.tmpdir}" />
851
        <mkdir dir="${junit.tmp.dir}/"/>
863
        <mkdir dir="${junit.tmp.dir}/"/>
852
        <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" fork="yes" maxmemory="2048m" dir="${basedir}" timeout="${test.timeout}" errorProperty="tests.failed" failureProperty="tests.failed">
864
        <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" fork="yes" maxmemory="2048m" dir="${basedir}" timeout="${test.timeout}" errorProperty="tests.failed" failureProperty="tests.failed">

    
   
865
            <sysproperty key="test.exec.type" value="${test.exec.type}" />
853
            <sysproperty key="ssh.gateway" value="${ssh.gateway}" />
866
            <sysproperty key="ssh.gateway" value="${ssh.gateway}" />
854
            <sysproperty key="hod.server" value="${hod.server}" />
867
            <sysproperty key="hod.server" value="${hod.server}" />
855
            <sysproperty key="java.io.tmpdir" value="${junit.tmp.dir}" />
868
            <sysproperty key="java.io.tmpdir" value="${junit.tmp.dir}" />
856
            <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
869
            <sysproperty key="hadoop.log.dir" value="${test.log.dir}"/>
857
            <jvmarg line="${debugArgs}"/>
870
            <jvmarg line="${debugArgs}"/>
[+20] [20] 816 lines
shims/test/hadoop20/org/apache/pig/test/MiniCluster.java
Revision d6384e9 New Change
 
shims/test/hadoop20/org/apache/pig/test/TezMiniCluster.java
Revision e69de29 New Change
 
shims/test/hadoop23/org/apache/pig/test/MiniCluster.java
Revision 1a4a430 New Change
 
shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java
Revision e69de29 New Change
 
test/tez-tests
Revision e69de29 New Change
 
test/org/apache/pig/test/MiniGenericCluster.java
Revision ac3f5bc New Change
 
test/org/apache/pig/test/Util.java
Revision 24103da New Change
 
test/org/apache/pig/tez/TestTezLauncher.java
Revision d4c3c6a New Change
 
  1. build.xml: Loading...
  2. shims/test/hadoop20/org/apache/pig/test/MiniCluster.java: Loading...
  3. shims/test/hadoop20/org/apache/pig/test/TezMiniCluster.java: Loading...
  4. shims/test/hadoop23/org/apache/pig/test/MiniCluster.java: Loading...
  5. shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java: Loading...
  6. test/tez-tests: Loading...
  7. test/org/apache/pig/test/MiniGenericCluster.java: Loading...
  8. test/org/apache/pig/test/Util.java: Loading...
  9. test/org/apache/pig/tez/TestTezLauncher.java: Loading...