Review Board 1.7.22


Add support for pulling HBase columns with prefixes

Review Request #9276 - Created Feb. 3, 2013 and updated

Swarnim Kulkarni
trunk
HIVE-3725
Reviewers
hive
hive-git
Added support for pulling hbase columns just by providing prefixes and a wildcard. So a query now could look something like this:

CREATE EXTERNAL TABLE hive_hbase_test
ROW FORMAT SERDE 'org.apache.hadoop.hive.hbase.HBaseSerDe' 
STORED BY 'org.apache.hadoop.hive.hbase.HBaseStorageHandler' 
WITH SERDEPROPERTIES ("hbase.columns.mapping" = ":key,fam1:col*") 
TBLPROPERTIES ("hbase.table.name" = "TEST_HBASE_TABLE");

This would pull in all columns under column family "fam1" which start with "col". This gives a little more flexibility over pull all columns format.
Added unit tests to demonstrate the new functionality. Also made sure that all existing unit tests passed.
hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java
Revision 7f37ba5 New Change
[20] 185 lines
[+20] [+] public void initialize(Configuration conf, Properties tbl)
186
        columnMapping.familyName = parts[0];
186
        columnMapping.familyName = parts[0];
187
        columnMapping.familyNameBytes = Bytes.toBytes(parts[0]);
187
        columnMapping.familyNameBytes = Bytes.toBytes(parts[0]);
188
        columnMapping.hbaseRowKey = false;
188
        columnMapping.hbaseRowKey = false;
189

    
   
189

   
190
        if (parts.length == 2) {
190
        if (parts.length == 2) {

    
   
191

   

    
   
192
          if (parts[1].endsWith("*")) {

    
   
193
            // we have a prefix with a wildcard

    
   
194
            columnMapping.qualifierPrefix = parts[1].substring(0, parts[1].length() - 1);

    
   
195
            columnMapping.qualifierPrefixBytes = Bytes.toBytes(columnMapping.qualifierPrefix);

    
   
196
            // we weren't provided any actual qualifier name. Set these to

    
   
197
            // null.

    
   
198
            columnMapping.qualifierName = null;

    
   
199
            columnMapping.qualifierNameBytes = null;

    
   
200
          } else {

    
   
201
            // set the regular provided qualifier names
191
          columnMapping.qualifierName = parts[1];
202
            columnMapping.qualifierName = parts[1];
192
          columnMapping.qualifierNameBytes = Bytes.toBytes(parts[1]);
203
            columnMapping.qualifierNameBytes = Bytes.toBytes(parts[1]);

    
   
204
            ;

    
   
205
          }
193
        } else {
206
        } else {
194
          columnMapping.qualifierName = null;
207
          columnMapping.qualifierName = null;
195
          columnMapping.qualifierNameBytes = null;
208
          columnMapping.qualifierNameBytes = null;
196
        }
209
        }
197
      }
210
      }
[+20] [20] 208 lines
[+20] [+] static class ColumnMapping {
406
    byte [] familyNameBytes;
419
    byte [] familyNameBytes;
407
    byte [] qualifierNameBytes;
420
    byte [] qualifierNameBytes;
408
    List<Boolean> binaryStorage;
421
    List<Boolean> binaryStorage;
409
    boolean hbaseRowKey;
422
    boolean hbaseRowKey;
410
    String mappingSpec;
423
    String mappingSpec;

    
   
424
    String qualifierPrefix;

    
   
425
    byte[] qualifierPrefixBytes;
411
  }
426
  }
412

    
   
427

   
413
  private void initHBaseSerDeParameters(
428
  private void initHBaseSerDeParameters(
414
      Configuration job, Properties tbl, String serdeName)
429
      Configuration job, Properties tbl, String serdeName)
415
    throws SerDeException {
430
    throws SerDeException {
[+20] [20] 403 lines
hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java
Revision a8ba9d9 New Change
 
hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java
Revision d35bb52 New Change
 
hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseSerDe.java
Revision e821282 New Change
 
  1. hbase-handler/src/java/org/apache/hadoop/hive/hbase/HBaseSerDe.java: Loading...
  2. hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseCellMap.java: Loading...
  3. hbase-handler/src/java/org/apache/hadoop/hive/hbase/LazyHBaseRow.java: Loading...
  4. hbase-handler/src/test/org/apache/hadoop/hive/hbase/TestHBaseSerDe.java: Loading...