Review Board 1.7.22


SQOOP-824 Sqoop code generation in 'update' export mode incompatible with '--columns' option

Review Request #8921 - Created Jan. 10, 2013 and submitted

Jarek Cecho
SQOOP-824
Reviewers
Sqoop
sqoop-trunk
I've updated ConnManager.configureDbOutputColumns() method to take into consideration --columns parameter.
I've provided unit tests to cover for new functionality.
src/java/org/apache/sqoop/manager/ConnManager.java
Revision 115186f50a6c16f074ad79b43a64644eefef04ac New Change
[20] 503 lines
[+20] [+] public void upsertTable(com.cloudera.sqoop.manager.ExportJobContext context)
504
    // We're in update mode. We need to explicitly set the database output
504
    // We're in update mode. We need to explicitly set the database output
505
    // column ordering in the codeGenerator.  The UpdateKeyCol must come
505
    // column ordering in the codeGenerator.  The UpdateKeyCol must come
506
    // last, because the UPDATE-based OutputFormat will generate the SET
506
    // last, because the UPDATE-based OutputFormat will generate the SET
507
    // clause followed by the WHERE clause, and the SqoopRecord needs to
507
    // clause followed by the WHERE clause, and the SqoopRecord needs to
508
    // serialize to this layout.
508
    // serialize to this layout.

    
   
509

   

    
   
510
    // Check if user specified --columns parameter

    
   
511
    Set<String> columns = null;

    
   
512
    if (options.getColumns() != null && options.getColumns().length > 0) {

    
   
513
      // If so, put all column in uppercase form into our help set

    
   
514
      columns = new HashSet<String>();

    
   
515
      for(String c : options.getColumns()) {

    
   
516
        columns.add(c.toUpperCase());

    
   
517
      }

    
   
518
    }

    
   
519

   
509
    Set<String> updateKeys = new LinkedHashSet<String>();
520
    Set<String> updateKeys = new LinkedHashSet<String>();
510
    Set<String> updateKeysUppercase = new HashSet<String>();
521
    Set<String> updateKeysUppercase = new HashSet<String>();
511
    String updateKeyValue = options.getUpdateKeyCol();
522
    String updateKeyValue = options.getUpdateKeyCol();
512
    StringTokenizer stok = new StringTokenizer(updateKeyValue, ",");
523
    StringTokenizer stok = new StringTokenizer(updateKeyValue, ",");
513
    while (stok.hasMoreTokens()) {
524
    while (stok.hasMoreTokens()) {
514
      String nextUpdateColumn = stok.nextToken().trim();
525
      String nextUpdateColumn = stok.nextToken().trim();
515
      if (nextUpdateColumn.length() > 0) {
526
      if (nextUpdateColumn.length() > 0) {

    
   
527
        String upperCase = nextUpdateColumn.toUpperCase();

    
   
528

   

    
   
529
        // We must make sure that --columns is super set of --update-key

    
   
530
        if (columns != null && !columns.contains(upperCase)) {

    
   
531
          throw new RuntimeException("You must specify all columns from "

    
   
532
            + "--update-key parameter in --columns parameter.");

    
   
533
        }

    
   
534

   
516
        updateKeys.add(nextUpdateColumn);
535
        updateKeys.add(nextUpdateColumn);
517
        updateKeysUppercase.add(nextUpdateColumn.toUpperCase());
536
        updateKeysUppercase.add(upperCase);
518
      } else {
537
      } else {
519
        throw new RuntimeException("Invalid update key column value specified"
538
        throw new RuntimeException("Invalid update key column value specified"
520
                    + ": '" + updateKeyValue + "'");
539
                    + ": '" + updateKeyValue + "'");
521
      }
540
      }
522
    }
541
    }
523
    String [] allColNames = getColumnNames(options.getTableName());
542
    String [] allColNames = getColumnNames(options.getTableName());
524
    List<String> dbOutCols = new ArrayList<String>();
543
    List<String> dbOutCols = new ArrayList<String>();
525
    for (String col : allColNames) {
544
    for (String col : allColNames) {
526
      if (!updateKeysUppercase.contains(col.toUpperCase())) {
545
      if (!updateKeysUppercase.contains(col.toUpperCase())) {

    
   
546
        // Skip columns that were not explicitly stated on command line

    
   
547
        if (columns != null && !columns.contains(col.toUpperCase())) {

    
   
548
          continue;

    
   
549
        }

    
   
550

   
527
        dbOutCols.add(col); // add non-key columns to the output order list.
551
        dbOutCols.add(col); // add non-key columns to the output order list.
528
      }
552
      }
529
    }
553
    }
530

    
   
554

   
531
    // Then add the update key column last.
555
    // Then add the update key column last.
[+20] [20] 57 lines
src/test/com/cloudera/sqoop/TestExportUpdate.java
Revision f5c30f348bab84c6ece49759e083b929097c9277 New Change
 
  1. src/java/org/apache/sqoop/manager/ConnManager.java: Loading...
  2. src/test/com/cloudera/sqoop/TestExportUpdate.java: Loading...