Review Board 1.7.22


HIVE-6208 user-defined aggregate functions cannot be used as windowing function

Review Request #16921 - Created Jan. 15, 2014 and updated

Jason Dere
HIVE-6208
Reviewers
hive
rhbutani
hive-git
- All UDAFs are also added to window function list when registered to function registry
- Avoid NPE when invalid function name used as window function
Added positive/negative test
ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java
Revision 96a78fc New Change
[20] 417 lines
[+20]
418

    
   
418

   
419
    //PTF declarations
419
    //PTF declarations
420
    registerGenericUDF(true, LEAD_FUNC_NAME, GenericUDFLead.class);
420
    registerGenericUDF(true, LEAD_FUNC_NAME, GenericUDFLead.class);
421
    registerGenericUDF(true, LAG_FUNC_NAME, GenericUDFLag.class);
421
    registerGenericUDF(true, LAG_FUNC_NAME, GenericUDFLag.class);
422

    
   
422

   
423
    registerHiveUDAFsAsWindowFunctions();

   
424
    registerWindowFunction("row_number", new GenericUDAFRowNumber());
423
    registerWindowFunction("row_number", new GenericUDAFRowNumber());
425
    registerWindowFunction("rank", new GenericUDAFRank());
424
    registerWindowFunction("rank", new GenericUDAFRank());
426
    registerWindowFunction("dense_rank", new GenericUDAFDenseRank());
425
    registerWindowFunction("dense_rank", new GenericUDAFDenseRank());
427
    registerWindowFunction("percent_rank", new GenericUDAFPercentRank());
426
    registerWindowFunction("percent_rank", new GenericUDAFPercentRank());
428
    registerWindowFunction("cume_dist", new GenericUDAFCumeDist());
427
    registerWindowFunction("cume_dist", new GenericUDAFCumeDist());
[+20] [20] 572 lines
[+20] [+] static void registerGenericUDAF(String functionName,
1001

    
   
1000

   
1002
  public static void registerGenericUDAF(boolean isNative, String functionName,
1001
  public static void registerGenericUDAF(boolean isNative, String functionName,
1003
      GenericUDAFResolver genericUDAFResolver) {
1002
      GenericUDAFResolver genericUDAFResolver) {
1004
    FunctionInfo fi = new FunctionInfo(isNative, functionName.toLowerCase(), genericUDAFResolver);
1003
    FunctionInfo fi = new FunctionInfo(isNative, functionName.toLowerCase(), genericUDAFResolver);
1005
    mFunctions.put(functionName.toLowerCase(), fi);
1004
    mFunctions.put(functionName.toLowerCase(), fi);

    
   
1005

   

    
   
1006
    // All aggregate functions should also be usable as window functions

    
   
1007
    addFunctionInfoToWindowFunctions(functionName, fi);

    
   
1008

   
1006
    registerNativeStatus(fi);
1009
    registerNativeStatus(fi);
1007
  }
1010
  }
1008

    
   
1011

   
1009
  public static void registerTemporaryUDAF(String functionName,
1012
  public static void registerTemporaryUDAF(String functionName,
1010
      Class<? extends UDAF> udafClass) {
1013
      Class<? extends UDAF> udafClass) {
[+20] [20] 8 lines
[+20] [+] static void registerUDAF(String functionName, Class<? extends UDAF> udafClass) {
1019
      Class<? extends UDAF> udafClass) {
1022
      Class<? extends UDAF> udafClass) {
1020
    FunctionInfo fi = new FunctionInfo(isNative,
1023
    FunctionInfo fi = new FunctionInfo(isNative,
1021
        functionName.toLowerCase(), new GenericUDAFBridge(
1024
        functionName.toLowerCase(), new GenericUDAFBridge(
1022
        (UDAF) ReflectionUtils.newInstance(udafClass, null)));
1025
        (UDAF) ReflectionUtils.newInstance(udafClass, null)));
1023
    mFunctions.put(functionName.toLowerCase(), fi);
1026
    mFunctions.put(functionName.toLowerCase(), fi);

    
   
1027

   

    
   
1028
    // All aggregate functions should also be usable as window functions

    
   
1029
    addFunctionInfoToWindowFunctions(functionName, fi);

    
   
1030

   
1024
    registerNativeStatus(fi);
1031
    registerNativeStatus(fi);
1025
  }
1032
  }
1026

    
   
1033

   
1027
  public static void unregisterTemporaryUDF(String functionName) throws HiveException {
1034
  public static void unregisterTemporaryUDF(String functionName) throws HiveException {
1028
    FunctionInfo fi = mFunctions.get(functionName.toLowerCase());
1035
    FunctionInfo fi = mFunctions.get(functionName.toLowerCase());
[+20] [20] 645 lines
[+20] [+] public static void registerWindowFunction(String name, GenericUDAFResolver wFn)
1674
   */
1681
   */
1675
  public static void registerWindowFunction(String name, GenericUDAFResolver wFn, boolean registerAsUDAF)
1682
  public static void registerWindowFunction(String name, GenericUDAFResolver wFn, boolean registerAsUDAF)
1676
  {
1683
  {
1677
    FunctionInfo fInfo = null;
1684
    FunctionInfo fInfo = null;
1678
    if (registerAsUDAF) {
1685
    if (registerAsUDAF) {

    
   
1686
      // Just register the function normally, will also get added to window functions.
1679
      registerGenericUDAF(true, name, wFn);
1687
      registerGenericUDAF(true, name, wFn);
1680
      fInfo = getFunctionInfo(name);

   
1681
    }
1688
    }
1682
    else {
1689
    else {
1683
      fInfo = new FunctionInfo(true,
1690
      name = name.toLowerCase();
1684
          name.toLowerCase(), wFn);
1691
      fInfo = new FunctionInfo(true, name, wFn);

    
   
1692
      addFunctionInfoToWindowFunctions(name, fInfo);
1685
    }
1693
    }
1686

    
   

   
1687
    WindowFunctionInfo wInfo = new WindowFunctionInfo(fInfo);

   
1688
    windowFunctions.put(name.toLowerCase(), wInfo);

   
1689
  }
1694
  }
1690

    
   
1695

   
1691
  public static WindowFunctionInfo getWindowFunctionInfo(String name)
1696
  public static WindowFunctionInfo getWindowFunctionInfo(String name)
1692
  {
1697
  {
1693
    return windowFunctions.get(name.toLowerCase());
1698
    return windowFunctions.get(name.toLowerCase());
[+20] [20] 23 lines
[+20] [+] public static boolean impliesOrder(String functionName) {
1717
      return windowInfo.isImpliesOrder();
1722
      return windowInfo.isImpliesOrder();
1718
    }
1723
    }
1719
    return false;
1724
    return false;
1720
  }
1725
  }
1721

    
   
1726

   
1722
  static void registerHiveUDAFsAsWindowFunctions()
1727
  static private void addFunctionInfoToWindowFunctions(String functionName,
1723
  {
1728
      FunctionInfo functionInfo) {
1724
    Set<String> fNames = getFunctionNames();
1729
    // Assumes that the caller has already verified that functionInfo is for an aggregate function
1725
    for(String fName : fNames)
1730
    WindowFunctionInfo wInfo = new WindowFunctionInfo(functionInfo);
1726
    {
1731
    windowFunctions.put(functionName.toLowerCase(), wInfo);
1727
      FunctionInfo fInfo = getFunctionInfo(fName);

   
1728
      if ( fInfo.isGenericUDAF())

   
1729
      {

   
1730
        WindowFunctionInfo wInfo = new WindowFunctionInfo(fInfo);

   
1731
        windowFunctions.put(fName, wInfo);

   
1732
      }

   
1733
    }

   
1734
  }
1732
  }
1735

    
   
1733

   
1736
  public static boolean isTableFunction(String name)
1734
  public static boolean isTableFunction(String name)
1737
  {
1735
  {
1738
    FunctionInfo tFInfo = mFunctions.get(name.toLowerCase());
1736
    FunctionInfo tFInfo = mFunctions.get(name.toLowerCase());
[+20] [20] 68 lines
ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java
Revision f011258 New Change
 
ql/src/test/queries/clientnegative/windowing_invalid_udaf.q
New File
 
ql/src/test/queries/clientpositive/windowing_udaf2.q
New File
 
ql/src/test/results/clientnegative/windowing_invalid_udaf.q.out
New File
 
ql/src/test/results/clientpositive/windowing_udaf2.q.out
New File
 
  1. ql/src/java/org/apache/hadoop/hive/ql/exec/FunctionRegistry.java: Loading...
  2. ql/src/java/org/apache/hadoop/hive/ql/parse/PTFTranslator.java: Loading...
  3. ql/src/test/queries/clientnegative/windowing_invalid_udaf.q: Loading...
  4. ql/src/test/queries/clientpositive/windowing_udaf2.q: Loading...
  5. ql/src/test/results/clientnegative/windowing_invalid_udaf.q.out: Loading...
  6. ql/src/test/results/clientpositive/windowing_udaf2.q.out: Loading...