Review Board 1.7.22


PIG-3190 Review

Review Request #9843 - Created March 11, 2013 and updated

Russell Jurney
trunk
PIG-3190
Reviewers
pig
ashutoshc, dvryaboy, haglein, jcoveney
pig-git
Adding Lucene Standard and Snowball tokenizers to Pig builtin
Unit tests and local testing.
ivy.xml
Revision 70e8d50 New Change
[20] 236 lines
[+20]
237
      <exclude org="com.sun.jersey" module="jersey-core"/>
237
      <exclude org="com.sun.jersey" module="jersey-core"/>
238
      <exclude org="com.sun.jersey" module="jersey-server"/>
238
      <exclude org="com.sun.jersey" module="jersey-server"/>
239
      <exclude org="com.sun.jersey" module="jersey-json"/>
239
      <exclude org="com.sun.jersey" module="jersey-json"/>
240
      <exclude org="asm" module="asm"/>
240
      <exclude org="asm" module="asm"/>
241
    </dependency>
241
    </dependency>

    
   
242
    

    
   
243
    <dependency org="org.apache.lucene" name="lucene-core" rev="4.1.0" conf="compile->master"/>

    
   
244
    <dependency org="org.apache.lucene" name="lucene-analyzers-common" rev="4.1.0" conf="compile->master"/>
242

    
   
245

   
243
    <!-- for TestHBaseStorage -->
246
    <!-- for TestHBaseStorage -->
244
    <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}"
247
    <dependency org="com.github.stephenc.high-scale-lib" name="high-scale-lib" rev="${high-scale-lib.version}"
245
       conf="test->default"/>
248
       conf="test->default"/>
246
    <dependency org="com.google.protobuf" name="protobuf-java" rev="${protobuf-java.version}"
249
    <dependency org="com.google.protobuf" name="protobuf-java" rev="${protobuf-java.version}"
[+20] [20] 18 lines
src/org/apache/pig/builtin/SnowballTokenize.java
New File
 
src/org/apache/pig/builtin/StandardTokenize.java
New File
 
test/org/apache/pig/test/TestSnowballTokenize.java
New File
 
test/org/apache/pig/test/TestStandardTokenize.java
New File
 
test/org/apache/pig/test/data/ExpectedSnowballTokens.txt
New File
 
test/org/apache/pig/test/data/ExpectedStandardTokens.txt
New File
 
test/org/apache/pig/test/data/InputFiles/ten_enron_emails.txt
New File
 
  1. ivy.xml: Loading...
  2. src/org/apache/pig/builtin/SnowballTokenize.java: Loading...
  3. src/org/apache/pig/builtin/StandardTokenize.java: Loading...
  4. test/org/apache/pig/test/TestSnowballTokenize.java: Loading...
  5. test/org/apache/pig/test/TestStandardTokenize.java: Loading...
  6. test/org/apache/pig/test/data/ExpectedSnowballTokens.txt: Loading...
  7. test/org/apache/pig/test/data/ExpectedStandardTokens.txt: Loading...
  8. test/org/apache/pig/test/data/InputFiles/ten_enron_emails.txt: Loading...