Review Board 1.7.22


ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java
New File

    
   
1
/**

    
   
2
 * Licensed to the Apache Software Foundation (ASF) under one

    
   
3
 * or more contributor license agreements.  See the NOTICE file

    
   
4
 * distributed with this work for additional information

    
   
5
 * regarding copyright ownership.  The ASF licenses this file

    
   
6
 * to you under the Apache License, Version 2.0 (the

    
   
7
 * "License"); you may not use this file except in compliance

    
   
8
 * with the License.  You may obtain a copy of the License at

    
   
9
 *

    
   
10
 *     http://www.apache.org/licenses/LICENSE-2.0

    
   
11
 *

    
   
12
 * Unless required by applicable law or agreed to in writing, software

    
   
13
 * distributed under the License is distributed on an "AS IS" BASIS,

    
   
14
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.

    
   
15
 * See the License for the specific language governing permissions and

    
   
16
 * limitations under the License.

    
   
17
 */

    
   
18

   

    
   
19
package org.apache.hadoop.hive.ql.exec.vector;

    
   
20

   

    
   
21
import java.util.List;

    
   
22

   

    
   
23
import org.apache.hadoop.hive.ql.metadata.HiveException;

    
   
24
import org.apache.hadoop.hive.serde2.io.DoubleWritable;

    
   
25
import org.apache.hadoop.hive.serde2.io.ShortWritable;

    
   
26
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;

    
   
27
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;

    
   
28
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;

    
   
29
import org.apache.hadoop.hive.serde2.objectinspector.StructField;

    
   
30
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;

    
   
31
import org.apache.hadoop.io.FloatWritable;

    
   
32
import org.apache.hadoop.io.IntWritable;

    
   
33
import org.apache.hadoop.io.LongWritable;

    
   
34
import org.apache.hadoop.io.Text;

    
   
35

   

    
   
36
public class VectorizedBatchUtil {

    
   
37

   

    
   
38
  /**

    
   
39
   * Sets the IsNull value for ColumnVector at specified index

    
   
40
   * @param cv

    
   
41
   * @param rowIndex

    
   
42
   */

    
   
43
  public static void SetNullColIsNullValue(ColumnVector cv, int rowIndex) {

    
   
44
    cv.isNull[rowIndex] = true;

    
   
45
    if (cv.noNulls) {

    
   
46
      cv.noNulls = false;

    
   
47
    }

    
   
48
  }

    
   
49

   

    
   
50
  /**

    
   
51
   * Iterates thru all the column vectors and sets noNull to

    
   
52
   * specified value.

    
   
53
   *

    
   
54
   * @param valueToSet

    
   
55
   *          noNull value to set

    
   
56
   * @param batch

    
   
57
   *          Batch on which noNull is set

    
   
58
   */

    
   
59
  public static void SetNoNullFields(boolean valueToSet, VectorizedRowBatch batch) {

    
   
60
    for (int i = 0; i < batch.numCols; i++) {

    
   
61
      batch.cols[i].noNulls = true;

    
   
62
    }

    
   
63
  }

    
   
64

   

    
   
65
  /**

    
   
66
   * Iterates thru all the columns in a given row and populates the batch

    
   
67
   * @param row Deserialized row object

    
   
68
   * @param oi Object insepector for that row

    
   
69
   * @param rowIndex index to which the row should be added to batch

    
   
70
   * @param batch Vectorized batch to which the row is added at rowIndex

    
   
71
   * @throws HiveException

    
   
72
   */

    
   
73
  public static void AddRowToBatch(Object row, StructObjectInspector oi, int rowIndex,

    
   
74
      VectorizedRowBatch batch) throws HiveException {

    
   
75
    List<? extends StructField> fieldRefs = oi.getAllStructFieldRefs();

    
   
76
    // Iterate thru the cols and load the batch

    
   
77
    for (int i = 0; i < fieldRefs.size(); i++) {

    
   
78
      Object fieldData = oi.getStructFieldData(row, fieldRefs.get(i));

    
   
79
      ObjectInspector foi = fieldRefs.get(i).getFieldObjectInspector();

    
   
80

   

    
   
81
      // Vectorization only supports PRIMITIVE data types. Assert the same

    
   
82
      assert (foi.getCategory() == Category.PRIMITIVE);

    
   
83

   

    
   
84
      // Get writable object

    
   
85
      PrimitiveObjectInspector poi = (PrimitiveObjectInspector) foi;

    
   
86
      Object writableCol = poi.getPrimitiveWritableObject(fieldData);

    
   
87

   

    
   
88
      // NOTE: The default value for null fields in vectorization is 1 for int types, NaN for

    
   
89
      // float/double. String types have no default value for null.

    
   
90
      switch (poi.getPrimitiveCategory()) {

    
   
91
      case SHORT: {

    
   
92
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];

    
   
93
        if (writableCol != null) {

    
   
94
          lcv.vector[rowIndex] = ((ShortWritable) writableCol).get();

    
   
95
          lcv.isNull[rowIndex] = false;

    
   
96
        } else {

    
   
97
          lcv.vector[rowIndex] = 1;

    
   
98
          SetNullColIsNullValue(lcv, rowIndex);

    
   
99
        }

    
   
100
      }

    
   
101
        break;

    
   
102
      case INT: {

    
   
103
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];

    
   
104
        if (writableCol != null) {

    
   
105
          lcv.vector[rowIndex] = ((IntWritable) writableCol).get();

    
   
106
          lcv.isNull[rowIndex] = false;

    
   
107
        } else {

    
   
108
          lcv.vector[rowIndex] = 1;

    
   
109
          SetNullColIsNullValue(lcv, rowIndex);

    
   
110
        }

    
   
111
      }

    
   
112
        break;

    
   
113
      case LONG: {

    
   
114
        LongColumnVector lcv = (LongColumnVector) batch.cols[i];

    
   
115
        if (writableCol != null) {

    
   
116
          lcv.vector[rowIndex] = ((LongWritable) writableCol).get();

    
   
117
          lcv.isNull[rowIndex] = false;

    
   
118
        } else {

    
   
119
          lcv.vector[rowIndex] = 1;

    
   
120
          SetNullColIsNullValue(lcv, rowIndex);

    
   
121
        }

    
   
122
      }

    
   
123
        break;

    
   
124
      case FLOAT: {

    
   
125
        DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[i];

    
   
126
        if (writableCol != null) {

    
   
127
          dcv.vector[rowIndex] = ((FloatWritable) writableCol).get();

    
   
128
          dcv.isNull[rowIndex] = false;

    
   
129
        } else {

    
   
130
          dcv.vector[rowIndex] = Double.NaN;

    
   
131
          SetNullColIsNullValue(dcv, rowIndex);

    
   
132
        }

    
   
133
      }

    
   
134
        break;

    
   
135
      case DOUBLE: {

    
   
136
        DoubleColumnVector dcv = (DoubleColumnVector) batch.cols[i];

    
   
137
        if (writableCol != null) {

    
   
138
          dcv.vector[rowIndex] = ((DoubleWritable) writableCol).get();

    
   
139
          dcv.isNull[rowIndex] = false;

    
   
140
        } else {

    
   
141
          dcv.vector[rowIndex] = Double.NaN;

    
   
142
          SetNullColIsNullValue(dcv, rowIndex);

    
   
143
        }

    
   
144
      }

    
   
145
        break;

    
   
146
      case STRING: {

    
   
147
        BytesColumnVector bcv = (BytesColumnVector) batch.cols[i];

    
   
148
        if (writableCol != null) {

    
   
149
          bcv.isNull[rowIndex] = false;

    
   
150
          Text colText = (Text) writableCol;

    
   
151
          bcv.setRef(rowIndex, colText.getBytes(), 0, colText.getLength());

    
   
152
        } else {

    
   
153
          SetNullColIsNullValue(bcv, rowIndex);

    
   
154
        }

    
   
155
      }

    
   
156
        break;

    
   
157
      default:

    
   
158
        throw new HiveException("Vectorizaton is not supported for datatype:"

    
   
159
            + poi.getPrimitiveCategory());

    
   
160
      }

    
   
161
    }

    
   
162
  }

    
   
163

   

    
   
164
}
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java
Revision 46256c4 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedSerde.java
Revision 78a9084 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/io/CommonRCFileInputFormat.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileInputFormat.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java
Revision 1306710 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java
Revision bfcc765 New Change
 
ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java
Revision 1bc6c74 New Change
 
serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java
Revision 11f5f07 New Change
 
serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java
Revision 27ed4ef New Change
 
  1. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedBatchUtil.java: Loading...
  2. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedColumnarSerDe.java: Loading...
  3. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedRowBatchCtx.java: Loading...
  4. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorizedSerde.java: Loading...
  5. ql/src/java/org/apache/hadoop/hive/ql/io/CommonRCFileInputFormat.java: Loading...
  6. ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileInputFormat.java: Loading...
  7. ql/src/java/org/apache/hadoop/hive/ql/io/VectorizedRCFileRecordReader.java: Loading...
  8. ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcSerde.java: Loading...
  9. ql/src/java/org/apache/hadoop/hive/ql/io/orc/VectorizedOrcInputFormat.java: Loading...
  10. ql/src/test/org/apache/hadoop/hive/ql/exec/vector/TestVectorizedRowBatchCtx.java: Loading...
  11. serde/src/java/org/apache/hadoop/hive/serde2/columnar/ColumnarSerDe.java: Loading...
  12. serde/src/java/org/apache/hadoop/hive/serde2/lazy/LazyUtils.java: Loading...