Review Board 1.7.22


SQOOP-600: HBase table family presence is not working correctly on new HBase versions

Review Request #7014 - Created Sept. 11, 2012 and submitted

Jarek Cecho
SQOOP-600
Reviewers
Sqoop
sqoop-trunk
Problem is that we're creating Table descriptor locally, which is not importing remote table families automatically. I've enhanced the code to retrieve remote table descriptor prior checking if the table family is present.
I've done only manual testing on HBase 0.94. Automatic tests are disabled in different profiles other than hadoopversion={20,100} and I'll work on getting them working later.

Diff revision 1 (Latest)

  1. src/java/org/apache/sqoop/mapreduce/HBaseImportJob.java: Loading...
src/java/org/apache/sqoop/mapreduce/HBaseImportJob.java
Revision 6a784d2 New Change
[20] 150 lines
[+20] [+] protected void jobSetup(Job job) throws IOException, ImportException {
151
      } else {
151
      } else {
152
        LOG.warn("Could not find HBase table " + tableName);
152
        LOG.warn("Could not find HBase table " + tableName);
153
        LOG.warn("This job may fail. Either explicitly create the table,");
153
        LOG.warn("This job may fail. Either explicitly create the table,");
154
        LOG.warn("or re-run with --hbase-create-table.");
154
        LOG.warn("or re-run with --hbase-create-table.");
155
      }
155
      }
156
    } else if (!tableDesc.hasFamily(familyBytes)) {
156
    } else {

    
   
157
      // Table exists, so retrieve their current version

    
   
158
      tableDesc = admin.getTableDescriptor(Bytes.toBytes(tableName));

    
   
159

   

    
   
160
      // Check if current version do have specified column family

    
   
161
      if (!tableDesc.hasFamily(familyBytes)) {
157
      if (options.getCreateHBaseTable()) {
162
        if (options.getCreateHBaseTable()) {
158
        // Create the column family.
163
          // Create the column family.
159
        LOG.info("Creating missing column family " + familyName);
164
          LOG.info("Creating missing column family " + familyName);
160
        admin.disableTable(tableName);
165
          admin.disableTable(tableName);
161
        admin.addColumn(tableName, colDesc);
166
          admin.addColumn(tableName, colDesc);
162
        admin.enableTable(tableName);
167
          admin.enableTable(tableName);
163
      } else {
168
        } else {
164
        LOG.warn("Could not find column family " + familyName + " in table "
169
          LOG.warn("Could not find column family " + familyName + " in table "
165
            + tableName);
170
            + tableName);
166
        LOG.warn("This job may fail. Either create the column family,");
171
          LOG.warn("This job may fail. Either create the column family,");
167
        LOG.warn("or re-run with --hbase-create-table.");
172
          LOG.warn("or re-run with --hbase-create-table.");
168
      }
173
        }
169
    }
174
      }

    
   
175
    }
170

    
   
176

   
171
    // Make sure HBase libraries are shipped as part of the job.
177
    // Make sure HBase libraries are shipped as part of the job.
172
    TableMapReduceUtil.addDependencyJars(job);
178
    TableMapReduceUtil.addDependencyJars(job);
173
    TableMapReduceUtil.addDependencyJars(conf, HTable.class);
179
    TableMapReduceUtil.addDependencyJars(conf, HTable.class);
174

    
   
180

   
175
    super.jobSetup(job);
181
    super.jobSetup(job);
176
  }
182
  }
177
}
183
}
178

    
   
184

   
  1. src/java/org/apache/sqoop/mapreduce/HBaseImportJob.java: Loading...