Review Board 1.7.22


Implement vectorized string column-scalar filters

Review Request #10785 - Created April 25, 2013 and updated

Eric Hanson
vectorization
HIVE-4383
Reviewers
hive
hive-git
Implement vectorized filters that compare a string column vector on the left and a scalar on the right. Use a template to generate code for all different comparison operators.

 
hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm
Revision 5940cbb New Change
[20] 65 lines
[+20]
66

    
   
66

   
67
#foreach ($qf in $qfiles)
67
#foreach ($qf in $qfiles)
68
  #set ($fname = $qf.getName())
68
  #set ($fname = $qf.getName())
69
  #set ($eidx = $fname.indexOf('.'))
69
  #set ($eidx = $fname.indexOf('.'))
70
  #set ($tname = $fname.substring(0, $eidx))
70
  #set ($tname = $fname.substring(0, $eidx))
71
  #set ($fpath = $qf.getCanonicalPath())
71
  #set ($fpath = $qf.getCanonicalPath().replaceAll("\\","\\\\"))

    
   
72
  public void testCliDriver_$tname() throws Exception {

    
   
73
    runTest("$tname", "$fname", "$fpath");

    
   
74
  }

    
   
75

   

    
   
76
#end

    
   
77

   

    
   
78
  private void runTest(String tname, String fname, String fpath) throws Exception {

    
   
79
    long startTime = System.currentTimeMillis();

    
   
80
    try {

    
   
81
      System.out.println("Begin query: " + fname);

    
   
82

   

    
   
83
      qt.addFile(fpath);

    
   
84

   

    
   
85
      if (qt.shouldBeSkipped(fname)) {

    
   
86
        System.out.println("Test " + fname + " skipped");

    
   
87
        return;

    
   
88
      }

    
   
89

   

    
   
90
      qt.cliInit(fname);

    
   
91
      qt.clearTestSideEffects();

    
   
92
      int ecode = qt.executeClient(fname);

    
   
93
      if (ecode == 0) {

    
   
94
        fail("Client Execution failed with error code = " + ecode);

    
   
95
      }

    
   
96

   

    
   
97
      ecode = qt.checkCliDriverResults(fname);

    
   
98
      if (ecode != 0) {

    
   
99
        fail("Client execution results failed with error code = " + ecode);

    
   
100
      }

    
   
101
      qt.clearPostTestEffects();

    
   
102

   

    
   
103
    } catch (Throwable e) {

    
   
104
      System.out.println("Exception: " + e.getMessage());

    
   
105
      e.printStackTrace();

    
   
106
      System.out.println("Failed query: " + fname);

    
   
107
      System.out.flush();

    
   
108
      fail("Unexpected exception");

    
   
109
    }

    
   
110

   

    
   
111
    long elapsedTime = System.currentTimeMillis() - startTime;

    
   
112
    System.out.println("Done query: " + fname + " elapsedTime=" + elapsedTime/1000 + "s");

    
   
113
    assertTrue("Test passed", true);

    
   
114
  }

    
   
115
}

    
   
116

   

    
   
117
package org.apache.hadoop.hive.cli;

    
   
118

   

    
   
119
import junit.framework.Test;

    
   
120
import junit.framework.TestCase;

    
   
121
import junit.framework.TestSuite;

    
   
122

   

    
   
123
import java.io.*;

    
   
124
import java.util.*;

    
   
125

   

    
   
126
import org.apache.hadoop.hive.hbase.HBaseQTestUtil;

    
   
127
import org.apache.hadoop.hive.hbase.HBaseTestSetup;

    
   
128

   

    
   
129
public class $className extends TestCase {

    
   
130

   

    
   
131
  private HBaseQTestUtil qt;

    
   
132
  private HBaseTestSetup setup;

    
   
133

   

    
   
134
  public $className(String name, HBaseTestSetup setup) {

    
   
135
    super(name);

    
   
136
    qt = null;

    
   
137
    this.setup = setup;

    
   
138
  }

    
   
139

   

    
   
140
  @Override

    
   
141
  protected void setUp() {

    
   
142
    try {

    
   
143
      boolean miniMR = false;

    
   
144
      if ("$clusterMode".equals("miniMR")) {

    
   
145
        miniMR = true;

    
   
146
      }

    
   
147

   

    
   
148
      qt = new HBaseQTestUtil("$resultsDir", "$logDir", miniMR, setup);

    
   
149

   

    
   
150
    } catch (Exception e) {

    
   
151
      System.out.println("Exception: " + e.getMessage());

    
   
152
      e.printStackTrace();

    
   
153
      System.out.flush();

    
   
154
      fail("Unexpected exception in setup");

    
   
155
    }

    
   
156
  }

    
   
157

   

    
   
158
  @Override

    
   
159
  protected void tearDown() {

    
   
160
    try {

    
   
161
      qt.shutdown();

    
   
162
    }

    
   
163
    catch (Exception e) {

    
   
164
      System.out.println("Exception: " + e.getMessage());

    
   
165
      e.printStackTrace();

    
   
166
      System.out.flush();

    
   
167
      fail("Unexpected exception in tearDown");

    
   
168
    }

    
   
169
  }

    
   
170

   

    
   
171
  public static Test suite() {

    
   
172
    TestSuite suite = new TestSuite();

    
   
173
    HBaseTestSetup setup = new HBaseTestSetup(suite);

    
   
174
#foreach ($qf in $qfiles)

    
   
175
  #set ($fname = $qf.getName())

    
   
176
  #set ($eidx = $fname.indexOf('.'))

    
   
177
  #set ($tname = $fname.substring(0, $eidx))

    
   
178
    suite.addTest(new $className("testCliDriver_$tname", setup));

    
   
179
#end

    
   
180
    return setup;

    
   
181
  }

    
   
182

   

    
   
183
#foreach ($qf in $qfiles)

    
   
184
  #set ($fname = $qf.getName())

    
   
185
  #set ($eidx = $fname.indexOf('.'))

    
   
186
  #set ($tname = $fname.substring(0, $eidx))

    
   
187
  #set ($fpath = $qf.getCanonicalPath().replaceAll("\\","\\\\"))
72
  public void testCliDriver_$tname() throws Exception {
188
  public void testCliDriver_$tname() throws Exception {
73
    runTest("$tname", "$fname", "$fpath");
189
    runTest("$tname", "$fname", "$fpath");
74
  }
190
  }
75

    
   
191

   
76
#end
192
#end
[+20] [20] 40 lines
hcatalog/build.xml
Revision 6f8389b New Change
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColEqualStringScalar.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColGreaterEqualStringScalar.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColGreaterStringScalar.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColLessEqualStringScalar.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColLessStringScalar.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColNotEqualStringScalar.java
New File
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java
Revision 32c7e11 New Change
 
ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareScalar.txt
New File
 
  1. hbase-handler/src/test/templates/TestHBaseNegativeCliDriver.vm: Loading...
  2. hcatalog/build.xml: Loading...
  3. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColEqualStringScalar.java: Loading...
  4. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColGreaterEqualStringScalar.java: Loading...
  5. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColGreaterStringScalar.java: Loading...
  6. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColLessEqualStringScalar.java: Loading...
  7. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColLessStringScalar.java: Loading...
  8. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/gen/FilterStringColNotEqualStringScalar.java: Loading...
  9. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/CodeGen.java: Loading...
  10. ql/src/java/org/apache/hadoop/hive/ql/exec/vector/expressions/templates/FilterStringColumnCompareScalar.txt: Loading...