Review Board 1.7.22


Adding JSONRecordReader

Review Request #11587 - Created May 31, 2013 and submitted

Timothy Chen
execwork
Reviewers
drill-git
jacques
drill-git
Added the JSONRecordReader based on the previous ScanJson work.
 Does not support nested fields, maps or lists yet.
 Currently it detects to move on to the next batch when any of the field batch cannot hold another item for the current item being written. This also assumes the default batch size can always hold at least one item from any field (which only is a problem for variable length vectors).

 

Diff revision 1

This is not the most recent revision of the diff. The latest diff is revision 5. See what's changed.

1 2 3 4 5
1 2 3 4 5

  1. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen4.java: Loading...
  2. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VariableVector.java: Loading...
  3. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java: Loading...
  4. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java: Loading...
  5. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/NamedField.java: Loading...
  6. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/OrderedField.java: Loading...
  7. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java: Loading...
  8. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperator.java: Loading...
  9. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperatorIterator.java: Loading...
  10. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/ScanJson.java: Loading...
  11. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java: Loading...
  12. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/BatchExceededException.java: Loading...
  13. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java: Loading...
  14. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java: Loading...
  15. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java: Loading...
  16. sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java: Loading...
  17. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json: Loading...
  18. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_2.json: Loading...
  19. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_3.json: Loading...
  20. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_4.json: Loading...
  21. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_5.json: Loading...
  22. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_6.json: Loading...
This diff has been split across 2 pages: < 1 2
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen4.java
Revision b3cd712 New Change
[20] 17 lines
[+20]
18
package org.apache.drill.exec.record.vector;
18
package org.apache.drill.exec.record.vector;
19

    
   
19

   
20
import org.apache.drill.exec.memory.BufferAllocator;
20
import org.apache.drill.exec.memory.BufferAllocator;
21
import org.apache.drill.exec.record.MaterializedField;
21
import org.apache.drill.exec.record.MaterializedField;
22

    
   
22

   

    
   
23
import static com.google.common.base.Preconditions.checkArgument;

    
   
24

   
23
public class VarLen4 extends VariableVector<VarLen4, Fixed4>{
25
public class VarLen4 extends VariableVector<VarLen4, Fixed4>{
24
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarLen4.class);
26
  static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(VarLen4.class);
25

    
   
27

   
26
  public VarLen4(MaterializedField field, BufferAllocator allocator) {
28
  public VarLen4(MaterializedField field, BufferAllocator allocator) {
27
    super(field, allocator);
29
    super(field, allocator, 4);
28
  }
30
  }
29

    
   
31

   
30
  @Override
32
  @Override
31
  protected Fixed4 getNewLengthVector(BufferAllocator allocator) {
33
  protected Fixed4 getNewLengthVector(BufferAllocator allocator) {
32
    return new Fixed4(null, allocator);
34
    return new Fixed4(null, allocator);
33
  }
35
  }
34
  
36

   
35
  
37
    public void setBytes(int index, byte[] bytes) {

    
   
38
        checkArgument(index >= 0);

    
   
39
        if(index == 0) {

    
   
40
            lengthVector.setInt(0, bytes.length);

    
   
41
            data.setBytes(0, bytes);

    
   
42
        } else {

    
   
43
            int previousOffset = lengthVector.getInt(index - 1);

    
   
44
            lengthVector.setInt(index, previousOffset + bytes.length);

    
   
45
            data.setBytes(previousOffset, bytes);

    
   
46
        }

    
   
47
    }
36
}
48
}
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VariableVector.java
Revision 4247f14 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java
Revision 016e097 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java
Revision e19c099 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/NamedField.java
Revision aa0d6aa New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/OrderedField.java
Revision 67fd2fa New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java
Revision 0643710 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperator.java
Revision e450ee9 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperatorIterator.java
Revision bf4053e New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/ScanJson.java
Revision a1c30e9 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
Revision d5aaab2 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/BatchExceededException.java
New File
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
New File
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
Revision 67c84ed New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
New File
 
sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_2.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_3.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_4.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_5.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_6.json
New File
 
  1. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VarLen4.java: Loading...
  2. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/record/vector/VariableVector.java: Loading...
  3. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/DiffSchema.java: Loading...
  4. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/Field.java: Loading...
  5. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/NamedField.java: Loading...
  6. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/OrderedField.java: Loading...
  7. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java: Loading...
  8. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperator.java: Loading...
  9. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperatorIterator.java: Loading...
  10. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/ScanJson.java: Loading...
  11. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java: Loading...
  12. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/BatchExceededException.java: Loading...
  13. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java: Loading...
  14. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java: Loading...
  15. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java: Loading...
  16. sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java: Loading...
  17. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json: Loading...
  18. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_2.json: Loading...
  19. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_3.json: Loading...
  20. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_4.json: Loading...
  21. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_5.json: Loading...
  22. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_6.json: Loading...
This diff has been split across 2 pages: < 1 2