Review Board 1.7.22


HIVE-5129: Multiple table insert fails on count(distinct)

Review Request #13697 - Created Aug. 21, 2013 and updated

Vikram Dixit Kumaraswamy
0.11.0
HIVE-5129
Reviewers
hive
navis
hive-git
Hive fails with class cast exception on multiple table insert fails on count(distinct).
Runs the test from HIVE-4692 successfully.
ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Revision 1d6e6fe New Change
[20] 7039 lines
[+20] [+] private Operator createCommonReduceSink(QB qb, Operator input)
7040
    }
7040
    }
7041

    
   
7041

   
7042
    List<Operator<? extends OperatorDesc>> inputOperators =
7042
    List<Operator<? extends OperatorDesc>> inputOperators =
7043
        new ArrayList<Operator<? extends OperatorDesc>>(ks.size());
7043
        new ArrayList<Operator<? extends OperatorDesc>>(ks.size());
7044
    List<List<ExprNodeDesc>> sprayKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size());
7044
    List<List<ExprNodeDesc>> sprayKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size());

    
   
7045
    List<List<ExprNodeDesc>> distinctKeyLists = new ArrayList<List<ExprNodeDesc>>(ks.size());
7045

    
   
7046

   
7046
    // Iterate over each clause
7047
    // Iterate over each clause
7047
    for (String dest : ks) {
7048
    for (String dest : ks) {
7048
      Operator input = inputs.get(dest);
7049
      Operator input = inputs.get(dest);
7049
      RowResolver inputRR = opParseCtx.get(input).getRowResolver();
7050
      RowResolver inputRR = opParseCtx.get(input).getRowResolver();
7050
      List<ExprNodeDesc> sprayKeys = getDistinctExprs(qbp, dest, inputRR);

   
7051

    
   
7051

   

    
   
7052
      List<ExprNodeDesc> distinctKeys = getDistinctExprs(qbp, dest, inputRR);

    
   
7053
      List<ExprNodeDesc> sprayKeys = new ArrayList<ExprNodeDesc>();

    
   
7054

   
7052
      // Add the group by expressions
7055
      // Add the group by expressions
7053
      List<ASTNode> grpByExprs = getGroupByForClause(qbp, dest);
7056
      List<ASTNode> grpByExprs = getGroupByForClause(qbp, dest);
7054
      for (ASTNode grpByExpr : grpByExprs) {
7057
      for (ASTNode grpByExpr : grpByExprs) {
7055
        ExprNodeDesc exprDesc = genExprNodeDesc(grpByExpr, inputRR);
7058
        ExprNodeDesc exprDesc = genExprNodeDesc(grpByExpr, inputRR);
7056
        if (ExprNodeDescUtils.indexOf(exprDesc, sprayKeys) < 0) {
7059
        if (ExprNodeDescUtils.indexOf(exprDesc, sprayKeys) < 0) {
[+20] [20] 5 lines
[+20] private Operator createCommonReduceSink(QB qb, Operator input)
7062
      boolean found = false;
7065
      boolean found = false;
7063
      for (int i = 0; i < sprayKeyLists.size(); i++) {
7066
      for (int i = 0; i < sprayKeyLists.size(); i++) {
7064
        if (!input.equals(inputOperators.get(i))) {
7067
        if (!input.equals(inputOperators.get(i))) {
7065
          continue;
7068
          continue;
7066
        }
7069
        }

    
   
7070

   

    
   
7071
        if (distinctKeys.isEmpty()) {

    
   
7072
          // current dest has no distinct keys.

    
   
7073
          List<ExprNodeDesc> combinedList = new ArrayList<ExprNodeDesc>();

    
   
7074
          combineExprNodeLists(sprayKeyLists.get(i), distinctKeyLists.get(i), combinedList);

    
   
7075
          if (!matchExprLists(combinedList, sprayKeys)) {

    
   
7076
            continue;

    
   
7077
          } // else do the common code at the end.

    
   
7078
        } else {

    
   
7079
          if (distinctKeyLists.get(i).isEmpty()) {

    
   
7080
            List<ExprNodeDesc> combinedList = new ArrayList<ExprNodeDesc>();

    
   
7081
            combineExprNodeLists(sprayKeys, distinctKeys, combinedList);

    
   
7082
            if (!matchExprLists(combinedList, sprayKeyLists.get(i))) {

    
   
7083
              continue;

    
   
7084
            } else {

    
   
7085
              // we have found a match. insert this distinct clause to head.

    
   
7086
              distinctKeyLists.remove(i);

    
   
7087
              sprayKeyLists.remove(i);

    
   
7088
              distinctKeyLists.add(i, distinctKeys);

    
   
7089
              sprayKeyLists.add(i, sprayKeys);

    
   
7090
              commonGroupByDestGroups.get(i).add(0, dest);

    
   
7091
              found = true;

    
   
7092
              break;

    
   
7093
            }

    
   
7094
          } else {

    
   
7095
            if (!matchExprLists(distinctKeyLists.get(i), distinctKeys)) {

    
   
7096
              continue;

    
   
7097
            }

    
   
7098

   
7067
        if (!matchExprLists(sprayKeyLists.get(i), sprayKeys)) {
7099
            if (!matchExprLists(sprayKeyLists.get(i), sprayKeys)) {
7068
          continue;
7100
              continue;
7069
        }
7101
            }

    
   
7102
            // else do common code

    
   
7103
          }

    
   
7104
        }
7070

    
   
7105

   

    
   
7106
        // common code
7071
        // A match was found, so add the clause to the corresponding list
7107
        // A match was found, so add the clause to the corresponding list
7072
        commonGroupByDestGroups.get(i).add(dest);
7108
        commonGroupByDestGroups.get(i).add(dest);
7073
        found = true;
7109
        found = true;
7074
        break;
7110
        break;
7075
      }
7111
      }
7076

    
   
7112

   
7077
      // No match was found, so create new entries
7113
      // No match was found, so create new entries
7078
      if (!found) {
7114
      if (!found) {
7079
        inputOperators.add(input);
7115
        inputOperators.add(input);
7080
        sprayKeyLists.add(sprayKeys);
7116
        sprayKeyLists.add(sprayKeys);

    
   
7117
        distinctKeyLists.add(distinctKeys);
7081
        List<String> destGroup = new ArrayList<String>();
7118
        List<String> destGroup = new ArrayList<String>();
7082
        destGroup.add(dest);
7119
        destGroup.add(dest);
7083
        commonGroupByDestGroups.add(destGroup);
7120
        commonGroupByDestGroups.add(destGroup);
7084
      }
7121
      }
7085
    }
7122
    }
7086

    
   
7123

   
7087
    return commonGroupByDestGroups;
7124
    return commonGroupByDestGroups;
7088
  }
7125
  }
7089

    
   
7126

   

    
   
7127
  private void combineExprNodeLists(List<ExprNodeDesc> list, List<ExprNodeDesc> list2,

    
   
7128
      List<ExprNodeDesc> combinedList) {

    
   
7129
    combinedList.addAll(list);

    
   
7130
    for (ExprNodeDesc elem : list2) {

    
   
7131
      if (!combinedList.contains(elem)) {

    
   
7132
        combinedList.add(elem);

    
   
7133
      }

    
   
7134
    }

    
   
7135
  }

    
   
7136

   
7090
  // Returns whether or not two lists contain the same elements independent of order
7137
  // Returns whether or not two lists contain the same elements independent of order
7091
  private boolean matchExprLists(List<ExprNodeDesc> list1, List<ExprNodeDesc> list2) {
7138
  private boolean matchExprLists(List<ExprNodeDesc> list1, List<ExprNodeDesc> list2) {
7092

    
   
7139

   
7093
    if (list1.size() != list2.size()) {
7140
    if (list1.size() != list2.size()) {
7094
      return false;
7141
      return false;
[+20] [20] 3197 lines
ql/src/test/queries/clientpositive/multi_insert_gby3.q
New File
 
ql/src/test/results/clientpositive/multi_insert_gby3.q.out
New File
 
  1. ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java: Loading...
  2. ql/src/test/queries/clientpositive/multi_insert_gby3.q: Loading...
  3. ql/src/test/results/clientpositive/multi_insert_gby3.q.out: Loading...