Review Board 1.7.22


Vectorized reader support for Byte Boolean and Timestamp.

Review Request #11350 - Created May 23, 2013 and updated

Sarvesh Sakalanaga
Hive-4511
Reviewers
hive
hive-git
Change adds support for datatypes byte and boolean for Vectorized query execution. 

 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
Revision 0e19e59 New Change
[20] 15 lines
[+20]
16
 * limitations under the License.
16
 * limitations under the License.
17
 */
17
 */
18

    
   
18

   
19
package org.apache.hadoop.hive.ql.exec.vector;
19
package org.apache.hadoop.hive.ql.exec.vector;
20

    
   
20

   

    
   
21
import java.sql.Timestamp;
21
import java.util.List;
22
import java.util.List;
22

    
   
23

   
23
import org.apache.hadoop.hive.ql.metadata.HiveException;
24
import org.apache.hadoop.hive.ql.metadata.HiveException;

    
   
25
import org.apache.hadoop.hive.serde2.io.ByteWritable;
24
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
26
import org.apache.hadoop.hive.serde2.io.DoubleWritable;
25
import org.apache.hadoop.hive.serde2.io.ShortWritable;
27
import org.apache.hadoop.hive.serde2.io.ShortWritable;

    
   
28
import org.apache.hadoop.hive.serde2.io.TimestampWritable;
26
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
29
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
27
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
30
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
28
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
31
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
29
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
32
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
30
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
33
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;

    
   
34
import org.apache.hadoop.io.BooleanWritable;
31
import org.apache.hadoop.io.FloatWritable;
35
import org.apache.hadoop.io.FloatWritable;
32
import org.apache.hadoop.io.IntWritable;
36
import org.apache.hadoop.io.IntWritable;
33
import org.apache.hadoop.io.LongWritable;
37
import org.apache.hadoop.io.LongWritable;
34
import org.apache.hadoop.io.Text;
38
import org.apache.hadoop.io.Text;
35

    
   
39

   
[+20] [20] 50 lines
[+20] [+] public static void AddRowToBatch(Object row, StructObjectInspector oi, int rowIndex,
86
      Object writableCol = poi.getPrimitiveWritableObject(fieldData);
90
      Object writableCol = poi.getPrimitiveWritableObject(fieldData);
87

    
   
91

   
88
      // NOTE: The default value for null fields in vectorization is 1 for int types, NaN for
92
      // NOTE: The default value for null fields in vectorization is 1 for int types, NaN for
89
      // float/double. String types have no default value for null.
93
      // float/double. String types have no default value for null.
90
      switch (poi.getPrimitiveCategory()) {
94
      switch (poi.getPrimitiveCategory()) {

    
   
95
      case BOOLEAN: {

    
   
96
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];

    
   
97
        if (writableCol != null) {

    
   
98
          lcv.vector[rowIndex] = ((BooleanWritable) writableCol).get() ? 1 : 0;

    
   
99
          lcv.isNull[rowIndex] = false;

    
   
100
        } else {

    
   
101
          lcv.vector[rowIndex] = 1;

    
   
102
          SetNullColIsNullValue(lcv, rowIndex);

    
   
103
        }

    
   
104
      }

    
   
105
        break;

    
   
106
      case BYTE: {

    
   
107
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];

    
   
108
        if (writableCol != null) {

    
   
109
          lcv.vector[rowIndex] = ((ByteWritable) writableCol).get();

    
   
110
          lcv.isNull[rowIndex] = false;

    
   
111
        } else {

    
   
112
          lcv.vector[rowIndex] = 1;

    
   
113
          SetNullColIsNullValue(lcv, rowIndex);

    
   
114
        }

    
   
115
      }

    
   
116
        break;
91
      case SHORT: {
117
      case SHORT: {
92
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];
118
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];
93
        if (writableCol != null) {
119
        if (writableCol != null) {
94
          lcv.vector[rowIndex] = ((ShortWritable) writableCol).get();
120
          lcv.vector[rowIndex] = ((ShortWritable) writableCol).get();
95
          lcv.isNull[rowIndex] = false;
121
          lcv.isNull[rowIndex] = false;
[+20] [20] 45 lines
[+20] public static void AddRowToBatch(Object row, StructObjectInspector oi, int rowIndex,
141
          dcv.vector[rowIndex] = Double.NaN;
167
          dcv.vector[rowIndex] = Double.NaN;
142
          SetNullColIsNullValue(dcv, rowIndex);
168
          SetNullColIsNullValue(dcv, rowIndex);
143
        }
169
        }
144
      }
170
      }
145
        break;
171
        break;

    
   
172
      case TIMESTAMP: {

    
   
173
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];

    
   
174
        if (writableCol != null) {

    
   
175
          Timestamp t = ((TimestampWritable) writableCol).getTimestamp();

    
   
176
          lcv.vector[rowIndex] = (t.getTime() * 1000000) + (t.getNanos() % 1000000);

    
   
177
          lcv.isNull[rowIndex] = false;

    
   
178
        } else {

    
   
179
          lcv.vector[rowIndex] = 1;

    
   
180
          SetNullColIsNullValue(lcv, rowIndex);

    
   
181
        }

    
   
182
      }

    
   
183
        break;
146
      case STRING: {
184
      case STRING: {
147
        BytesColumnVector bcv = (BytesColumnVector) batch.cols[i];
185
        BytesColumnVector bcv = (BytesColumnVector) batch.cols[i];
148
        if (writableCol != null) {
186
        if (writableCol != null) {
149
          bcv.isNull[rowIndex] = false;
187
          bcv.isNull[rowIndex] = false;
150
          Text colText = (Text) writableCol;
188
          Text colText = (Text) writableCol;
[+20] [20] 14 lines
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java
Revision 676b86a New Change
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
Revision 1a57766 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java
Revision 11c3fdf New Change
 
ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java
Revision f15779d New Change
 
ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java
Revision 678a2f5 New Change
 
ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java
Revision 90d3134 New Change
 
ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java
Revision e8c75c6 New Change
 
serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
Revision 6958284 New Change
 
  1. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java: Loading...
  2. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java: Loading...
  3. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java: Loading...
  4. ql/src/java/org/apache/hadoop/hive/ql/io/orc/BitFieldReader.java: Loading...
  5. ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java: Loading...
  6. ql/src/java/org/apache/hadoop/hive/ql/io/orc/RunLengthByteReader.java: Loading...
  7. ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java: Loading...
  8. ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestVectorizedORCReader.java: Loading...
  9. serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java: Loading...