Review Board 1.7.22


SQOOP-654 PostgreSQL direct connector is ignoring --null(-input)string and --null(-input)-non-string arguments

Review Request #9204 - Created Jan. 31, 2013 and submitted

Jarek Cecho
SQOOP-654
Reviewers
Sqoop
sqoop-trunk
This patches fixes the NULL handling in PostgreSQL direct connector.
Added new test case that covers this eventuality.
src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java
Revision dba732c963416202934cb95b86eac274a3e1a2dd New Change
[20] 27 lines
[+20]
28
import java.io.OutputStreamWriter;
28
import java.io.OutputStreamWriter;
29
import java.util.ArrayList;
29
import java.util.ArrayList;
30
import java.util.List;
30
import java.util.List;
31
import java.util.Map;
31
import java.util.Map;
32

    
   
32

   

    
   
33
import org.apache.commons.lang.StringUtils;
33
import org.apache.commons.logging.Log;
34
import org.apache.commons.logging.Log;
34
import org.apache.commons.logging.LogFactory;
35
import org.apache.commons.logging.LogFactory;
35
import org.apache.sqoop.util.PostgreSQLUtils;
36
import org.apache.sqoop.util.PostgreSQLUtils;
36

    
   
37

   
37
import com.cloudera.sqoop.SqoopOptions;
38
import com.cloudera.sqoop.SqoopOptions;
[+20] [20] 5 lines
[+20]
43
import com.cloudera.sqoop.util.Executor;
44
import com.cloudera.sqoop.util.Executor;
44
import com.cloudera.sqoop.util.ImportException;
45
import com.cloudera.sqoop.util.ImportException;
45
import com.cloudera.sqoop.util.JdbcUrl;
46
import com.cloudera.sqoop.util.JdbcUrl;
46
import com.cloudera.sqoop.util.LoggingAsyncSink;
47
import com.cloudera.sqoop.util.LoggingAsyncSink;
47
import com.cloudera.sqoop.util.PerfCounters;
48
import com.cloudera.sqoop.util.PerfCounters;

    
   
49
import org.apache.sqoop.util.SubstitutionUtils;
48

    
   
50

   
49
/**
51
/**
50
 * Manages direct dumps from Postgresql databases via psql COPY TO STDOUT
52
 * Manages direct dumps from Postgresql databases via psql COPY TO STDOUT
51
 * commands.
53
 * commands.
52
 */
54
 */
[+20] [20] 202 lines
[+20] [+] private String getCopyCommand(String tableName) {
255

    
   
257

   
256
    // Translate delimiter characters to '\ooo' octal representation.
258
    // Translate delimiter characters to '\ooo' octal representation.
257
    sb.append(" TO STDOUT WITH DELIMITER E'\\");
259
    sb.append(" TO STDOUT WITH DELIMITER E'\\");
258
    sb.append(Integer.toString((int) this.options.getOutputFieldDelim(), 8));
260
    sb.append(Integer.toString((int) this.options.getOutputFieldDelim(), 8));
259
    sb.append("' CSV ");
261
    sb.append("' CSV ");

    
   
262

   

    
   
263
    if (this.options.getNullStringValue() != null) {

    
   
264
      sb.append("NULL AS E'");

    
   
265
      sb.append(SubstitutionUtils.removeEscapeCharacters(

    
   
266
        this.options.getNullStringValue()));

    
   
267
      sb.append("' ");

    
   
268
    }

    
   
269

   
260
    if (this.options.getOutputEnclosedBy() != '\0') {
270
    if (this.options.getOutputEnclosedBy() != '\0') {
261
      sb.append("QUOTE E'\\");
271
      sb.append("QUOTE E'\\");
262
      sb.append(Integer.toString((int) this.options.getOutputEnclosedBy(), 8));
272
      sb.append(Integer.toString((int) this.options.getOutputEnclosedBy(), 8));
263
      sb.append("' ");
273
      sb.append("' ");
264
    }
274
    }
[+20] [20] 10 lines
[+20] private String getCopyCommand(String tableName) {
275
    }
285
    }
276

    
   
286

   
277
    sb.append(";");
287
    sb.append(";");
278

    
   
288

   
279
    String copyCmd = sb.toString();
289
    String copyCmd = sb.toString();
280
    LOG.debug("Copy command is " + copyCmd);
290
    LOG.info("Copy command is " + copyCmd);
281
    return copyCmd;
291
    return copyCmd;
282
  }
292
  }
283

    
   
293

   
284
  /** Write the COPY command to a temp file.
294
  /** Write the COPY command to a temp file.
285
    * @return the filename we wrote to.
295
    * @return the filename we wrote to.
[+20] [20] 29 lines
[+20] [+] public void importTable(com.cloudera.sqoop.manager.ImportJobContext context)
315
      LOG.warn("File import layout" + options.getFileLayout()
325
      LOG.warn("File import layout" + options.getFileLayout()
316
          + " is not supported by");
326
          + " is not supported by");
317
      LOG.warn("Postgresql direct import; import will proceed as text files.");
327
      LOG.warn("Postgresql direct import; import will proceed as text files.");
318
    }
328
    }
319

    
   
329

   

    
   
330
    if (!StringUtils.equals(options.getNullStringValue(),

    
   
331
      options.getNullNonStringValue())) {

    
   
332
      throw new ImportException(

    
   
333
        "Detected different values of --input-string and --input-non-string " +

    
   
334
        "parameters. PostgreSQL direct manager do not support that. Please " +

    
   
335
        "either use the same values or omit the --direct parameter.");

    
   
336
    }

    
   
337

   
320
    String commandFilename = null;
338
    String commandFilename = null;
321
    String passwordFilename = null;
339
    String passwordFilename = null;
322
    Process p = null;
340
    Process p = null;
323
    AsyncSink sink = null;
341
    AsyncSink sink = null;
324
    AsyncSink errSink = null;
342
    AsyncSink errSink = null;
[+20] [20] 169 lines
src/java/org/apache/sqoop/util/SubstitutionUtils.java
New File
 
src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java
Revision 512b1d5566e6aad221e9b8b8c15be1e6037ad4f2 New Change
 
src/test/com/cloudera/sqoop/util/TestSubstitutionUtils.java
New File
 
  1. src/java/org/apache/sqoop/manager/DirectPostgresqlManager.java: Loading...
  2. src/java/org/apache/sqoop/util/SubstitutionUtils.java: Loading...
  3. src/test/com/cloudera/sqoop/manager/PostgresqlImportTest.java: Loading...
  4. src/test/com/cloudera/sqoop/util/TestSubstitutionUtils.java: Loading...