Review Board 1.7.22


HIVE-2272: add TIMESTAMP data type

Review Request #1135 - Created July 15, 2011 and updated

Franklin Hu
HIVE-2272
Reviewers
hive
heyongqiang, nzhang, sdong
hive
Adds TIMESTAMP type to serde2 with both string (LazySimple) and binary (LazyBinary) serialization.
Supports SQL style jdbc timestamps of the format with nanosecond precision
YYYY-MM-DD HH:MM:SS[.fff...]
Wrote some unit tests for string/binary serialization. Also ran against test data and looked consistent
trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
Revision 1156396 New Change
[20] 152 lines
[+20]
153
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapAnd;
153
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapAnd;
154
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapEmpty;
154
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapEmpty;
155
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapOr;
155
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFEWAHBitmapOr;
156
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFElt;
156
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFElt;
157
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFField;
157
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFField;

    
   
158
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFFromUtcTimestamp;
158
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
159
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFHash;
159
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf;
160
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIf;
160
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
161
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIn;
161
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex;
162
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFIndex;
162
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInstr;
163
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFInstr;
[+20] [20] 16 lines
[+20]
179
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSentences;
180
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSentences;
180
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSize;
181
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSize;
181
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSplit;
182
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFSplit;
182
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStringToMap;
183
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStringToMap;
183
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;
184
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFStruct;

    
   
185
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFTimestamp;

    
   
186
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFToUtcTimestamp;
184
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnion;
187
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFUnion;
185
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
188
import org.apache.hadoop.hive.ql.udf.generic.GenericUDFWhen;
186
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
189
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTF;
187
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode;
190
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFExplode;
188
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFJSONTuple;
191
import org.apache.hadoop.hive.ql.udf.generic.GenericUDTFJSONTuple;
[+20] [20] 177 lines
[+20]
366
    registerUDF(Constants.DOUBLE_TYPE_NAME, UDFToDouble.class, false,
369
    registerUDF(Constants.DOUBLE_TYPE_NAME, UDFToDouble.class, false,
367
        UDFToDouble.class.getSimpleName());
370
        UDFToDouble.class.getSimpleName());
368
    registerUDF(Constants.STRING_TYPE_NAME, UDFToString.class, false,
371
    registerUDF(Constants.STRING_TYPE_NAME, UDFToString.class, false,
369
        UDFToString.class.getSimpleName());
372
        UDFToString.class.getSimpleName());
370

    
   
373

   

    
   
374
    registerGenericUDF(Constants.TIMESTAMP_TYPE_NAME,

    
   
375
        GenericUDFTimestamp.class);

    
   
376

   
371
    // Aggregate functions
377
    // Aggregate functions
372
    registerGenericUDAF("max", new GenericUDAFMax());
378
    registerGenericUDAF("max", new GenericUDAFMax());
373
    registerGenericUDAF("min", new GenericUDAFMin());
379
    registerGenericUDAF("min", new GenericUDAFMin());
374

    
   
380

   
375
    registerGenericUDAF("sum", new GenericUDAFSum());
381
    registerGenericUDAF("sum", new GenericUDAFSum());
[+20] [20] 42 lines
[+20]
418
    registerGenericUDF("array_contains", GenericUDFArrayContains.class);
424
    registerGenericUDF("array_contains", GenericUDFArrayContains.class);
419
    registerGenericUDF("sentences", GenericUDFSentences.class);
425
    registerGenericUDF("sentences", GenericUDFSentences.class);
420
    registerGenericUDF("map_keys", GenericUDFMapKeys.class);
426
    registerGenericUDF("map_keys", GenericUDFMapKeys.class);
421
    registerGenericUDF("map_values", GenericUDFMapValues.class);
427
    registerGenericUDF("map_values", GenericUDFMapValues.class);
422

    
   
428

   

    
   
429
    registerGenericUDF("from_utc_timestamp", GenericUDFFromUtcTimestamp.class);

    
   
430
    registerGenericUDF("to_utc_timestamp", GenericUDFToUtcTimestamp.class);

    
   
431

   

    
   
432

   
423
    // Generic UDTF's
433
    // Generic UDTF's
424
    registerGenericUDTF("explode", GenericUDTFExplode.class);
434
    registerGenericUDTF("explode", GenericUDTFExplode.class);
425
    registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class);
435
    registerGenericUDTF("json_tuple", GenericUDTFJSONTuple.class);
426
    registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class);
436
    registerGenericUDTF("parse_url_tuple", GenericUDTFParseUrlTuple.class);
427
  }
437
  }
[+20] [20] 192 lines
[+20] [+] public static TypeInfo getCommonClassForComparison(TypeInfo a, TypeInfo b) {
620
   * The common class of string and double is string.
630
   * The common class of string and double is string.
621
   *
631
   *
622
   * @return null if no common class could be found.
632
   * @return null if no common class could be found.
623
   */
633
   */
624
  public static TypeInfo getCommonClass(TypeInfo a, TypeInfo b) {
634
  public static TypeInfo getCommonClass(TypeInfo a, TypeInfo b) {

    
   
635
    if (a.equals(b)) {

    
   
636
      return a;

    
   
637
    }
625
    Integer ai = numericTypes.get(a);
638
    Integer ai = numericTypes.get(a);
626
    Integer bi = numericTypes.get(b);
639
    Integer bi = numericTypes.get(b);
627
    if (ai == null || bi == null) {
640
    if (ai == null || bi == null) {
628
      // If either is not a numeric type, return null.
641
      // If either is not a numeric type, return null.
629
      return null;
642
      return null;
[+20] [20] 17 lines
[+20] [+] public static boolean implicitConvertable(TypeInfo from, TypeInfo to) {
647
    // Void can be converted to any type
660
    // Void can be converted to any type
648
    if (from.equals(TypeInfoFactory.voidTypeInfo)) {
661
    if (from.equals(TypeInfoFactory.voidTypeInfo)) {
649
      return true;
662
      return true;
650
    }
663
    }
651

    
   
664

   

    
   
665
    if (from.equals(TypeInfoFactory.timestampTypeInfo)

    
   
666
        && to.equals(TypeInfoFactory.stringTypeInfo)) {

    
   
667
      return true;

    
   
668
    }

    
   
669

   
652
    // Allow implicit conversion from Byte -> Integer -> Long -> Float -> Double
670
    // Allow implicit conversion from Byte -> Integer -> Long -> Float -> Double
653
    // -> String
671
    // -> String
654
    Integer f = numericTypes.get(from);
672
    Integer f = numericTypes.get(from);
655
    Integer t = numericTypes.get(to);
673
    Integer t = numericTypes.get(to);
656
    if (f == null || t == null) {
674
    if (f == null || t == null) {
[+20] [20] 448 lines
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java
Revision 1156396 New Change
 
trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java
Revision 1156396 New Change
 
  1. trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java: Loading...
  2. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java: Loading...
  3. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/ErrorMsg.java: Loading...
  4. trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/TypeCheckProcFactory.java: Loading...
  5. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDate.java: Loading...
  6. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateAdd.java: Loading...
  7. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateDiff.java: Loading...
  8. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDateSub.java: Loading...
  9. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFDayOfMonth.java: Loading...
  10. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFHour.java: Loading...
  11. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMinute.java: Loading...
  12. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFMonth.java: Loading...
  13. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFSecond.java: Loading...
  14. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToBoolean.java: Loading...
  15. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToByte.java: Loading...
  16. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToDouble.java: Loading...
  17. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToFloat.java: Loading...
  18. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToInteger.java: Loading...
  19. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToLong.java: Loading...
  20. trunk/ql/src/java/org/apache/hadoop/hive/ql/udf/UDFToShort.java: Loading...
This diff has been split across 4 pages: 1 2 3 4 >