Review Board 1.7.22


Adding JSONRecordReader

Review Request #11587 - Created May 31, 2013 and submitted

Timothy Chen
execwork
Reviewers
drill-git
jacques
drill-git
Added the JSONRecordReader based on the previous ScanJson work.
 Does not support nested fields, maps or lists yet.
 Currently it detects to move on to the next batch when any of the field batch cannot hold another item for the current item being written. This also assumes the default batch size can always hold at least one item from any field (which only is a problem for variable length vectors).

 

Diff revision 3

This is not the most recent revision of the diff. The latest diff is revision 5. See what's changed.

1 2 3 4 5
1 2 3 4 5

  1. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/OrderedField.java: Loading...
  2. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/SchemaRecorder.java: Loading...
  3. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java: Loading...
  4. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperator.java: Loading...
  5. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperatorIterator.java: Loading...
  6. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/ScanJson.java: Loading...
  7. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/transform/ProtobufSchemaTransformer.java: Loading...
  8. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/transform/SchemaTransformer.java: Loading...
  9. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java: Loading...
  10. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/BatchExceededException.java: Loading...
  11. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java: Loading...
  12. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java: Loading...
  13. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java: Loading...
  14. sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java: Loading...
  15. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json: Loading...
  16. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_2.json: Loading...
  17. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_3.json: Loading...
  18. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_4.json: Loading...
  19. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_5.json: Loading...
  20. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_6.json: Loading...
  21. sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/MultiLevelMap.java: Loading...
  22. sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/WindowingROP.java: Loading...
  23. sandbox/prototype/exec/ref/src/test/resources/donuts.json: Loading...
  24. sandbox/prototype/exec/ref/src/test/resources/simple_plan.json: Loading...
This diff has been split across 2 pages: < 1 2
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/OrderedField.java
Revision 67fd2fa New Change
[20] 16 lines
[+20]
17
 ******************************************************************************/
17
 ******************************************************************************/
18

    
   
18

   
19
package org.apache.drill.exec.schema;
19
package org.apache.drill.exec.schema;
20

    
   
20

   
21
import com.google.common.base.Objects;
21
import com.google.common.base.Objects;

    
   
22
import org.apache.drill.exec.proto.SchemaDefProtos;
22

    
   
23

   
23
public class OrderedField extends Field {
24
public class OrderedField extends Field {
24
    public OrderedField(RecordSchema parentSchema, IdGenerator<Integer> generator, FieldType fieldType, String prefixFieldName, int index) {
25
    private final int index;
25
        super(parentSchema, generator, fieldType, prefixFieldName);
26

   
26
        this.fieldName = "[" + index + "]";
27
    public OrderedField(RecordSchema parentSchema,

    
   
28
                        int parentFieldId,

    
   
29
                        IdGenerator<Integer> generator,

    
   
30
                        SchemaDefProtos.MajorType type,

    
   
31
                        String prefixFieldName,

    
   
32
                        int index) {

    
   
33
        super(parentSchema, parentFieldId, generator, type, prefixFieldName);

    
   
34
        this.index = index;

    
   
35
    }

    
   
36

   

    
   
37
    @Override

    
   
38
    public String getFieldName() {

    
   
39
        return "[" + index + "]";
27
    }
40
    }
28

    
   
41

   
29
    @Override
42
    @Override
30
    protected Objects.ToStringHelper addAttributesToHelper(Objects.ToStringHelper helper) {
43
    protected Objects.ToStringHelper addAttributesToHelper(Objects.ToStringHelper helper) {
31
        return helper;
44
        return helper;
32
    }
45
    }
33
}
46
}
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/SchemaRecorder.java
Revision 54a4e0e New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java
Revision 0643710 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperator.java
Revision e450ee9 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperatorIterator.java
Revision bf4053e New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/ScanJson.java
Revision a1c30e9 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/transform/ProtobufSchemaTransformer.java
Revision a81a9d9 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/transform/SchemaTransformer.java
Revision 54d851f New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java
Revision d5aaab2 New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/BatchExceededException.java
New File
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java
New File
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java
Revision 67c84ed New Change
 
sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java
New File
 
sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_2.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_3.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_4.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_5.json
New File
 
sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_6.json
New File
 
sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/MultiLevelMap.java
New File
 
sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/WindowingROP.java
New File
 
sandbox/prototype/exec/ref/src/test/resources/donuts.json
Revision 2d98b59 New Change
 
sandbox/prototype/exec/ref/src/test/resources/simple_plan.json
Revision 0f38476 New Change
 
  1. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/OrderedField.java: Loading...
  2. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/SchemaRecorder.java: Loading...
  3. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/JacksonHelper.java: Loading...
  4. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperator.java: Loading...
  5. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/PhysicalOperatorIterator.java: Loading...
  6. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/json/jackson/ScanJson.java: Loading...
  7. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/transform/ProtobufSchemaTransformer.java: Loading...
  8. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/schema/transform/SchemaTransformer.java: Loading...
  9. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/server/DrillbitContext.java: Loading...
  10. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/BatchExceededException.java: Loading...
  11. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/JSONRecordReader.java: Loading...
  12. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/RecordReader.java: Loading...
  13. sandbox/prototype/exec/java-exec/src/main/java/org/apache/drill/exec/store/VectorHolder.java: Loading...
  14. sandbox/prototype/exec/java-exec/src/test/java/org/apache/drill/exec/store/JSONRecordReaderTest.java: Loading...
  15. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_1.json: Loading...
  16. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_2.json: Loading...
  17. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_3.json: Loading...
  18. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_4.json: Loading...
  19. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_5.json: Loading...
  20. sandbox/prototype/exec/java-exec/src/test/resources/scan_json_test_6.json: Loading...
  21. sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/MultiLevelMap.java: Loading...
  22. sandbox/prototype/exec/ref/src/main/java/org/apache/drill/exec/ref/rops/WindowingROP.java: Loading...
  23. sandbox/prototype/exec/ref/src/test/resources/donuts.json: Loading...
  24. sandbox/prototype/exec/ref/src/test/resources/simple_plan.json: Loading...
This diff has been split across 2 pages: < 1 2