Review Board 1.7.22


Changes to add support for streaming_python udfs.

Review Request #13781 - Created Aug. 23, 2013 and updated

Jeremy Karn
Reviewers
pig
pig-git
Changes for PIG-2417 (https://issues.apache.org/jira/browse/PIG-2417)

 

Diff revision 3 (Latest)

1 2 3
1 2 3

  1. build.xml: Loading...
  2. src/org/apache/pig/impl/builtin/StreamingUDF.java: Loading...
  3. src/org/apache/pig/impl/streaming/ExecutableManager.java: Loading...
  4. src/org/apache/pig/impl/streaming/OutputHandler.java: Loading...
  5. src/org/apache/pig/impl/streaming/PigToStreamUDF.java: Loading...
  6. src/org/apache/pig/impl/streaming/StreamUDFToPig.java: Loading...
  7. src/org/apache/pig/impl/streaming/StreamingDelimiters.java: Loading...
  8. src/org/apache/pig/impl/streaming/StreamingUDFException.java: Loading...
  9. src/org/apache/pig/impl/streaming/StreamingUDFInputHandler.java: Loading...
  10. src/org/apache/pig/impl/streaming/StreamingUDFOutputHandler.java: Loading...
  11. src/org/apache/pig/impl/streaming/StreamingUDFOutputSchemaException.java: Loading...
  12. src/org/apache/pig/impl/streaming/StreamingUtil.java: Loading...
  13. src/org/apache/pig/impl/util/JarManager.java: Loading...
  14. src/org/apache/pig/impl/util/StorageUtil.java: Loading...
  15. src/org/apache/pig/scripting/ScriptEngine.java: Loading...
  16. src/org/apache/pig/scripting/ScriptingOutputCapturer.java: Loading...
  17. src/org/apache/pig/scripting/streaming/python/PythonScriptEngine.java: Loading...
  18. src/python/streaming/controller.py: Loading...
  19. src/python/streaming/pig_util.py: Loading...
  20. test/unit-tests: Loading...
  21. test/org/apache/pig/impl/builtin/TestStreamingUDF.java: Loading...
  22. test/org/apache/pig/impl/streaming/TestExecutableManager.java: Loading...
  23. test/org/apache/pig/impl/streaming/TestPigToStreamUDF.java: Loading...
  24. test/org/apache/pig/impl/streaming/TestStreamUDFToPig.java: Loading...
  25. test/org/apache/pig/impl/streaming/TestStreamingUDFOutputHandler.java: Loading...
  26. test/org/apache/pig/impl/streaming/TestStreamingUtil.java: Loading...
  27. test/org/apache/pig/test/TestPigStreaming.java: Loading...
  28. test/python/streaming/test_controller.py: Loading...
build.xml
Revision 7e22192 New Change
[20] 37 lines
[+20]
38
    <property name="year" value="2007-2012" />
38
    <property name="year" value="2007-2012" />
39

    
   
39

   
40
    <!-- source properties -->
40
    <!-- source properties -->
41
    <property name="lib.dir" value="${basedir}/lib" />
41
    <property name="lib.dir" value="${basedir}/lib" />
42
    <property name="src.dir" value="${basedir}/src" />
42
    <property name="src.dir" value="${basedir}/src" />

    
   
43
    <property name="python.src.dir" value="${src.dir}/python" />
43
    <property name="src.lib.dir" value="${basedir}/lib-src" />
44
    <property name="src.lib.dir" value="${basedir}/lib-src" />
44
    <property name="src.gen.dir" value="${basedir}/src-gen" />
45
    <property name="src.gen.dir" value="${basedir}/src-gen" />
45
    <property name="docs.dir" value="${basedir}/src/docs" />
46
    <property name="docs.dir" value="${basedir}/src/docs" />
46

    
   
47

   
47

    
   
48

   
[+20] [20] 522 lines
[+20]
570
                <compilerarg line="${javac.args} @{warnings}"/>
571
                <compilerarg line="${javac.args} @{warnings}"/>
571
                <classpath refid="@{cp}" />
572
                <classpath refid="@{cp}" />
572
            </javac>
573
            </javac>
573
            <copy file="${src.dir}/org/apache/pig/tools/grunt/autocomplete" todir="${build.classes}/org/apache/pig/tools/grunt"/>
574
            <copy file="${src.dir}/org/apache/pig/tools/grunt/autocomplete" todir="${build.classes}/org/apache/pig/tools/grunt"/>
574
            <copy file="${src.dir}/org/apache/pig/tools/grunt/autocomplete_aliases" todir="${build.classes}/org/apache/pig/tools/grunt"/>
575
            <copy file="${src.dir}/org/apache/pig/tools/grunt/autocomplete_aliases" todir="${build.classes}/org/apache/pig/tools/grunt"/>

    
   
576
            <copy todir="${build.classes}/python">

    
   
577
                <fileset dir="${python.src.dir}"/>

    
   
578
            </copy>
575
        </sequential>
579
        </sequential>
576
    </macrodef>
580
    </macrodef>
577

    
   
581

   
578
    <!-- ================================================================== -->
582
    <!-- ================================================================== -->
579
    <!-- Documentation                                                      -->
583
    <!-- Documentation                                                      -->
[+20] [20] 58 lines
[+20]
638
                    <attribute name="Implementation-Version" value="${version}" />
642
                    <attribute name="Implementation-Version" value="${version}" />
639
                </section>
643
                </section>
640
            </manifest>
644
            </manifest>
641
            <fileset dir="${src.lib.dir}/shock"/>
645
            <fileset dir="${src.lib.dir}/shock"/>
642
            <fileset dir="${src.lib.dir}/bzip2"/>
646
            <fileset dir="${src.lib.dir}/bzip2"/>

    
   
647
            <fileset dir="${python.src.dir}"/>
643
        </jar>
648
        </jar>
644
    </target>
649
    </target>
645
    <target name="javadoc-jar" depends="cc-compile, javadoc">
650
    <target name="javadoc-jar" depends="cc-compile, javadoc">
646
        <jar duplicate="preserve" jarfile="${output.jarfile.javadoc}" basedir="${build.javadoc}/">
651
        <jar duplicate="preserve" jarfile="${output.jarfile.javadoc}" basedir="${build.javadoc}/">
647
            <manifest>
652
            <manifest>
[+20] [20] 1002 lines
src/org/apache/pig/impl/builtin/StreamingUDF.java
New File
 
src/org/apache/pig/impl/streaming/ExecutableManager.java
Revision cf79c83 New Change
 
src/org/apache/pig/impl/streaming/OutputHandler.java
Revision 6e9262a New Change
 
src/org/apache/pig/impl/streaming/PigToStreamUDF.java
New File
 
src/org/apache/pig/impl/streaming/StreamUDFToPig.java
New File
 
src/org/apache/pig/impl/streaming/StreamingDelimiters.java
New File
 
src/org/apache/pig/impl/streaming/StreamingUDFException.java
New File
 
src/org/apache/pig/impl/streaming/StreamingUDFInputHandler.java
New File
 
src/org/apache/pig/impl/streaming/StreamingUDFOutputHandler.java
New File
 
src/org/apache/pig/impl/streaming/StreamingUDFOutputSchemaException.java
New File
 
src/org/apache/pig/impl/streaming/StreamingUtil.java
New File
 
src/org/apache/pig/impl/util/JarManager.java
Revision 5c4acb0 New Change
 
src/org/apache/pig/impl/util/StorageUtil.java
Revision dcb62ec New Change
 
src/org/apache/pig/scripting/ScriptEngine.java
Revision 29a9e1f New Change
 
src/org/apache/pig/scripting/ScriptingOutputCapturer.java
New File
 
src/org/apache/pig/scripting/streaming/python/PythonScriptEngine.java
New File
 
src/python/streaming/controller.py
New File
 
src/python/streaming/pig_util.py
New File
 
test/unit-tests
Revision d52ad9d New Change
 
test/org/apache/pig/impl/builtin/TestStreamingUDF.java
New File
 
test/org/apache/pig/impl/streaming/TestExecutableManager.java
Revision 6246019 New Change
 
test/org/apache/pig/impl/streaming/TestPigToStreamUDF.java
New File
 
test/org/apache/pig/impl/streaming/TestStreamUDFToPig.java
New File
 
test/org/apache/pig/impl/streaming/TestStreamingUDFOutputHandler.java
New File
 
test/org/apache/pig/impl/streaming/TestStreamingUtil.java
New File
 
test/org/apache/pig/test/TestPigStreaming.java
New File
 
test/python/streaming/test_controller.py
New File
 
  1. build.xml: Loading...
  2. src/org/apache/pig/impl/builtin/StreamingUDF.java: Loading...
  3. src/org/apache/pig/impl/streaming/ExecutableManager.java: Loading...
  4. src/org/apache/pig/impl/streaming/OutputHandler.java: Loading...
  5. src/org/apache/pig/impl/streaming/PigToStreamUDF.java: Loading...
  6. src/org/apache/pig/impl/streaming/StreamUDFToPig.java: Loading...
  7. src/org/apache/pig/impl/streaming/StreamingDelimiters.java: Loading...
  8. src/org/apache/pig/impl/streaming/StreamingUDFException.java: Loading...
  9. src/org/apache/pig/impl/streaming/StreamingUDFInputHandler.java: Loading...
  10. src/org/apache/pig/impl/streaming/StreamingUDFOutputHandler.java: Loading...
  11. src/org/apache/pig/impl/streaming/StreamingUDFOutputSchemaException.java: Loading...
  12. src/org/apache/pig/impl/streaming/StreamingUtil.java: Loading...
  13. src/org/apache/pig/impl/util/JarManager.java: Loading...
  14. src/org/apache/pig/impl/util/StorageUtil.java: Loading...
  15. src/org/apache/pig/scripting/ScriptEngine.java: Loading...
  16. src/org/apache/pig/scripting/ScriptingOutputCapturer.java: Loading...
  17. src/org/apache/pig/scripting/streaming/python/PythonScriptEngine.java: Loading...
  18. src/python/streaming/controller.py: Loading...
  19. src/python/streaming/pig_util.py: Loading...
  20. test/unit-tests: Loading...
  21. test/org/apache/pig/impl/builtin/TestStreamingUDF.java: Loading...
  22. test/org/apache/pig/impl/streaming/TestExecutableManager.java: Loading...
  23. test/org/apache/pig/impl/streaming/TestPigToStreamUDF.java: Loading...
  24. test/org/apache/pig/impl/streaming/TestStreamUDFToPig.java: Loading...
  25. test/org/apache/pig/impl/streaming/TestStreamingUDFOutputHandler.java: Loading...
  26. test/org/apache/pig/impl/streaming/TestStreamingUtil.java: Loading...
  27. test/org/apache/pig/test/TestPigStreaming.java: Loading...
  28. test/python/streaming/test_controller.py: Loading...