Review Board 1.7.22


Initialize object inspectors with union of table properties and partition properties

Review Request #11632 - Created June 4, 2013 and updated

Mark Wagner
Reviewers
hive
ashutoshc
hive-git
Change the initialization of object inspectors and deserializers to use the union of partition properties and table properties for partitioned tables. There is no change for unpartitioned tables.
I've done manual end-to-end testing with various queries/tables and have created a .q test for reading partitioned Avro tables.
ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java
Revision 9422bf7 New Change
[20] 239 lines
[+20] [+] private MapOpCtx initObjectInspector(MapredWork conf,
240
          throws HiveException,
240
          throws HiveException,
241
      ClassNotFoundException, InstantiationException, IllegalAccessException,
241
      ClassNotFoundException, InstantiationException, IllegalAccessException,
242
      SerDeException {
242
      SerDeException {
243
    PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
243
    PartitionDesc pd = conf.getPathToPartitionInfo().get(onefile);
244
    LinkedHashMap<String, String> partSpec = pd.getPartSpec();
244
    LinkedHashMap<String, String> partSpec = pd.getPartSpec();
245
    // Use tblProps in case of unpartitioned tables
245
    // Use table properties in case of unpartitioned tables,

    
   
246
    // and the union of table properties and partition properties, with partition

    
   
247
    // taking precedence
246
    Properties partProps =
248
    Properties partProps =
247
        (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
249
        (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
248
            pd.getTableDesc().getProperties() : pd.getProperties();
250
            pd.getTableDesc().getProperties() : pd.getOverlayedProperties();
249

    
   
251

   
250
    Class serdeclass = pd.getDeserializerClass();
252
    Class serdeclass = pd.getDeserializerClass();
251
    if (serdeclass == null) {
253
    if (serdeclass == null) {
252
      String className = pd.getSerdeClassName();
254
      String className = pd.getSerdeClassName();
253
      if ((className == null) || (className.isEmpty())) {
255
      if ((className == null) || (className.isEmpty())) {
[+20] [20] 153 lines
[+20] [+] private void setInspectorInput(MapInputPath inp) {
407
        TableDesc tableDesc = pd.getTableDesc();
409
        TableDesc tableDesc = pd.getTableDesc();
408
        Properties tblProps = tableDesc.getProperties();
410
        Properties tblProps = tableDesc.getProperties();
409
        // If the partition does not exist, use table properties
411
        // If the partition does not exist, use table properties
410
        Properties partProps =
412
        Properties partProps =
411
            (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
413
            (pd.getPartSpec() == null || pd.getPartSpec().isEmpty()) ?
412
                tblProps : pd.getProperties();
414
                tblProps : pd.getOverlayedProperties();
413

    
   
415

   
414
        Class sdclass = pd.getDeserializerClass();
416
        Class sdclass = pd.getDeserializerClass();
415
        if (sdclass == null) {
417
        if (sdclass == null) {
416
          String className = pd.getSerdeClassName();
418
          String className = pd.getSerdeClassName();
417
          if ((className == null) || (className.isEmpty())) {
419
          if ((className == null) || (className.isEmpty())) {
[+20] [20] 333 lines
ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java
Revision f0b16e4 New Change
 
ql/src/test/queries/clientpositive/avro_partitioned.q
New File
 
ql/src/test/results/clientpositive/avro_partitioned.q.out
New File
 
  1. ql/src/java/org/apache/hadoop/hive/ql/exec/MapOperator.java: Loading...
  2. ql/src/java/org/apache/hadoop/hive/ql/plan/PartitionDesc.java: Loading...
  3. ql/src/test/queries/clientpositive/avro_partitioned.q: Loading...
  4. ql/src/test/results/clientpositive/avro_partitioned.q.out: Loading...