Review Board 1.7.22


This addresses SQOOP-929

Review Request #9846 - Created March 11, 2013 and updated

Venkat Ranganathan
SQOOP-929
Reviewers
Sqoop
jarcec
sqoop-trunk
Expanded netezza tests 
Ran against NZ VM and all tests pass.  Checkstyle validated with no new regressions introduced
src/test/com/cloudera/sqoop/manager/DirectNetezzaExportManualTest.java
Revision bbcd138 New Change
[20] 15 lines
[+20]
16
 * limitations under the License.
16
 * limitations under the License.
17
 */
17
 */
18

    
   
18

   
19
package com.cloudera.sqoop.manager;
19
package com.cloudera.sqoop.manager;
20

    
   
20

   
21
import java.io.BufferedWriter;

   
22
import java.io.IOException;
21
import java.io.IOException;
23
import java.io.OutputStream;

   
24
import java.io.OutputStreamWriter;

   
25
import java.sql.Connection;

   
26
import java.sql.PreparedStatement;

   
27
import java.sql.ResultSet;

   
28
import java.sql.Statement;

   
29
import java.sql.SQLException;
22
import java.sql.SQLException;

    
   
23
import java.util.Arrays;
30

    
   
24

   
31
import org.apache.commons.logging.Log;
25
import org.apache.commons.logging.Log;
32
import org.apache.commons.logging.LogFactory;
26
import org.apache.commons.logging.LogFactory;
33
import org.apache.hadoop.conf.Configuration;

   
34
import org.apache.hadoop.fs.FileSystem;

   
35
import org.apache.hadoop.fs.Path;

   
36
import org.apache.sqoop.manager.DirectNetezzaManager;
27
import org.apache.sqoop.manager.DirectNetezzaManager;
37
import org.junit.After;
28
import org.junit.After;
38
import org.junit.Before;
29
import org.junit.Before;

    
   
30
import org.junit.Test;

    
   
31

   
39

    
   
32

   
40
import com.cloudera.sqoop.SqoopOptions;
33
import com.cloudera.sqoop.SqoopOptions;
41
import com.cloudera.sqoop.TestExport;
34
import com.cloudera.sqoop.TestExport.ColumnGenerator;
42

    
   
35

   
43
/**
36
/**
44
 * Test the DirectNetezzaManager implementation's exportJob() functionality.
37
 * Test the DirectNetezzaManager implementation's exportJob() functionality.
45
 */
38
 */
46
public class DirectNetezzaExportManualTest extends TestExport {
39
public class DirectNetezzaExportManualTest extends NetezzaExportManualTest {
47

    
   

   
48
  public static final Log LOG = LogFactory.getLog(

   
49
      DirectNetezzaExportManualTest.class.getName());

   
50

    
   

   
51
  static final String TABLE_PREFIX = "EMPNZ";

   
52

    
   

   
53
  // instance variables populated during setUp, used during tests.

   
54
  private DirectNetezzaManager manager;

   
55
  private Connection conn;

   
56

    
   
40

   
57
  @Override
41
  public static final Log LOG = LogFactory
58
  protected Connection getConnection() {
42
      .getLog(DirectNetezzaExportManualTest.class.getName());
59
    return conn;

   
60
  }

   
61

    
   

   
62
  @Override

   
63
  protected boolean useHsqldbTestServer() {

   
64
    return false;

   
65
  }

   
66

    
   
43

   
67
  @Override
44
  static final String TABLE_PREFIX = "EMPNZ_D_EXP";
68
  protected String getConnectString() {

   
69
    return NetezzaTestUtils.getNZConnectString();

   
70
  }

   
71

    
   
45

   
72
  @Override
46
  @Override
73
  protected String getTablePrefix() {
47
  protected String getTablePrefix() {
74
    return TABLE_PREFIX;
48
    return TABLE_PREFIX;
75
  }
49
  }
76

    
   
50

   
77
  @Override
51
  @Override
78
  protected String getDropTableStatement(String tableName) {
52
  protected boolean isDirectMode() {
79
    return "DROP TABLE " + tableName;
53
    return true;
80
  }

   
81

    
   

   
82
  @Before

   
83
  public void setUp() {

   
84
    super.setUp();

   
85
    conn = getConnection();

   
86
    SqoopOptions options = new SqoopOptions(

   
87
        NetezzaTestUtils.getNZConnectString(), getTableName());

   
88
    options.setUsername(NetezzaTestUtils.getNZUser());

   
89
    options.setPassword(NetezzaTestUtils.getNZPassword());

   
90
    this.manager = new DirectNetezzaManager(options);

   
91

    
   

   
92
    try {

   
93
      this.conn = manager.getConnection();

   
94
      this.conn.setAutoCommit(false);

   
95
    } catch (SQLException sqlE) {

   
96
      LOG.error("Encountered SQL Exception: " + sqlE);

   
97
      sqlE.printStackTrace();

   
98
      fail("SQLException when running test setUp(): " + sqlE);

   
99
    }

   
100
  }

   
101

    
   

   
102
  @After

   
103
  public void tearDown() {

   
104
    super.tearDown();

   
105
    if (null != manager) {

   
106
      try {

   
107
        manager.close();

   
108
      } catch (SQLException sqlE) {

   
109
        LOG.error("Got SQLException: " + sqlE.toString());

   
110
        fail("Got SQLException: " + sqlE.toString());

   
111
      }

   
112
    }

   
113
    this.conn = null;

   
114
    this.manager = null;

   
115

    
   

   
116
  }
54
  }
117

    
   
55

   
118
  @Override
56
  @Override
119
  protected String [] getCodeGenArgv(String... extraArgs) {
57
  protected String[] getCodeGenArgv(String... extraArgs) {
120

    
   
58

   
121
    String [] moreArgs = new String[extraArgs.length + 4];
59
    String[] moreArgs = new String[extraArgs.length + 4];
122
    int i = 0;
60
    int i = 0;
123
    for (i = 0; i < extraArgs.length; i++) {
61
    for (i = 0; i < extraArgs.length; i++) {
124
      moreArgs[i] = extraArgs[i];
62
      moreArgs[i] = extraArgs[i];
125
    }
63
    }
126

    
   
64

   
[+20] [20] 5 lines
[+20] public void tearDown() { protected String[] getCodeGenArgv(String... extraArgs) {
132

    
   
70

   
133
    return super.getCodeGenArgv(moreArgs);
71
    return super.getCodeGenArgv(moreArgs);
134
  }
72
  }
135

    
   
73

   
136
  @Override
74
  @Override
137
  protected String [] getArgv(boolean includeHadoopFlags,
75
  protected String[] getArgv(boolean includeHadoopFlags, int rowsPerStatement,
138
      int rowsPerStatement, int statementsPerTx, String... additionalArgv) {
76
      int statementsPerTx, String... additionalArgv) {
139

    
   
77

   
140
    String [] subArgv = newStrArray(additionalArgv, "--direct",
78
    String[] argV = super.getArgv(includeHadoopFlags,
141
        "--username", NetezzaTestUtils.getNZUser(), "--password",
79
        rowsPerStatement, statementsPerTx);

    
   
80
    String[] subArgV = newStrArray(argV, "--direct",

    
   
81
      "--username",  NetezzaTestUtils.getNZUser(), "--password",
142
        NetezzaTestUtils.getNZPassword());
82
      NetezzaTestUtils.getNZPassword());
143
    return super.getArgv(includeHadoopFlags, rowsPerStatement,
83
    String[] newArgV = new String[subArgV.length + additionalArgv.length];
144
        statementsPerTx, subArgv);
84
    int i = 0;

    
   
85
    for (String s : subArgV) {

    
   
86
      newArgV[i++] = s;

    
   
87
    }

    
   
88
    for (String s: additionalArgv) {

    
   
89
      newArgV[i++] = s;

    
   
90
    }

    
   
91
    return newArgV;
145
  }
92
  }
146

    
   
93

   
147

    
   
94

   
148

    
   

   
149
  /**
95
  /**
150
   * Create the table definition to export to, removing any prior table. By
96
   * Create the table definition to export to, removing any prior table. By
151
   * specifying ColumnGenerator arguments, you can add extra columns to the
97
   * specifying ColumnGenerator arguments, you can add extra columns to the
152
   * table of arbitrary type.
98
   * table of arbitrary type.
153
   */
99
   */
154
  @Override
100
  @Override
155
  public void createTable(ColumnGenerator... extraColumns) throws SQLException {
101
  public void createTable(ColumnGenerator... extraColumns)
156
    PreparedStatement statement = conn.prepareStatement(
102
    throws SQLException {
157
        getDropTableStatement(getTableName()), ResultSet.TYPE_FORWARD_ONLY,
103
    createTableNZ(getTableName(), extraColumns);
158
        ResultSet.CONCUR_READ_ONLY);

   
159
    try {

   
160
      statement.executeUpdate();

   
161
      conn.commit();

   
162
    } catch (SQLException sqle) {

   
163
      conn.rollback();

   
164
    } finally {

   
165
      statement.close();

   
166
    }

   
167

    
   

   
168
    StringBuilder sb = new StringBuilder();

   
169
    sb.append("CREATE TABLE ");

   
170
    sb.append(getTableName());

   
171
    sb.append(" (id INT NOT NULL PRIMARY KEY, msg VARCHAR(64)");

   
172
    int colNum = 0;

   
173
    for (ColumnGenerator gen : extraColumns) {

   
174
      sb.append(", " + forIdx(colNum++) + " " + gen.getType());

   
175
    }
104
  }
176
    sb.append(")");

   
177

    
   
105

   
178
    statement = conn.prepareStatement(sb.toString(),

   
179
        ResultSet.TYPE_FORWARD_ONLY, ResultSet.CONCUR_READ_ONLY);

   
180
    try {

   
181
      statement.executeUpdate();

   
182
      conn.commit();

   
183
    } finally {

   
184
      statement.close();

   
185
    }

   
186
  }

   
187
  /**
106
  /**
188
   * Test an authenticated export using netezza external table import.
107
   * Creates the staging table.

    
   
108
   * @param extraColumns extra columns that go in the staging table

    
   
109
   * @throws SQLException if an error occurs during export
189
   */
110
   */
190
  public void testAuthExport() throws IOException, SQLException {
111
  @Override

    
   
112
  public void createStagingTable(ColumnGenerator... extraColumns)

    
   
113
    throws SQLException {

    
   
114
    createTableNZ(getStagingTableName(), extraColumns);

    
   
115
  }

    
   
116

   

    
   
117
  private void runNetezzaTest(String tableName, String[] argv,

    
   
118
    ColumnGenerator...extraCols) throws IOException {
191
    SqoopOptions options = new SqoopOptions(
119
    SqoopOptions options = new SqoopOptions(
192
        NetezzaTestUtils.getNZConnectString(),
120
        NetezzaTestUtils.getNZConnectString(), getTableName());
193
        getTableName());

   
194
    options.setUsername(NetezzaTestUtils.getNZUser());
121
    options.setUsername(NetezzaTestUtils.getNZUser());
195
    options.setPassword(NetezzaTestUtils.getNZPassword());
122
    options.setPassword(NetezzaTestUtils.getNZPassword());
196

    
   
123

   

    
   
124
    LOG.info("Running export with argv : " + Arrays.toString(argv));
197
    manager = new DirectNetezzaManager(options);
125
    manager = new DirectNetezzaManager(options);
198

    
   
126

   
199
    Connection connection = null;

   
200
    Statement st = null;

   
201

    
   

   
202
    String tableName = getTableName();

   
203

    
   

   
204
    try {

   
205
      connection = manager.getConnection();

   
206
      connection.setAutoCommit(false);

   
207
      st = connection.createStatement();

   
208

    
   

   
209
      // create a target database table.

   
210
      try {
127
    try {
211
        st.executeUpdate("DROP TABLE " + tableName);
128
      createTable(extraCols);
212
      } catch(SQLException sqle) {

   
213
        LOG.info("Ignoring exception from DROP TABLE : " + sqle.getMessage());

   
214
        connection.rollback();

   
215
      }

   
216

    
   

   
217
      LOG.info("Creating table " + tableName);

   
218

    
   

   
219
      st.executeUpdate("CREATE TABLE " + tableName + " ("

   
220
          + "id INT NOT NULL PRIMARY KEY, "

   
221
          + "msg VARCHAR(24) NOT NULL)");

   
222

    
   

   
223
      connection.commit();

   
224
      LOG.info("Created table " + tableName);
129
      LOG.info("Created table " + tableName);
225

    
   
130
      createExportFile(extraCols);
226
      // Write a file containing a record to export.

   
227
      Path tablePath = getTablePath();

   
228
      Path filePath = new Path(tablePath, "datafile");

   
229
      Configuration conf = new Configuration();

   
230

    
   

   
231
      FileSystem fs = FileSystem.get(conf);

   
232
      fs.mkdirs(tablePath);

   
233
      OutputStream os = fs.create(filePath);

   
234
      BufferedWriter w = new BufferedWriter(new OutputStreamWriter(os));

   
235
      w.write(getRecordLine(0));

   
236
      w.write(getRecordLine(1));

   
237
      w.write(getRecordLine(2));

   
238
      w.close();

   
239
      os.close();

   
240

    
   

   
241
      // run the export and verify that the results are good.
131
      // run the export and verify that the results are good.
242
      runExport(getArgv(true, 10, 10,
132
      runExport(argv);
243
          "--username", NetezzaTestUtils.getNZUser(),
133
      verifyExport(3, conn);
244
          "--password", NetezzaTestUtils.getNZPassword(),
134
      if (extraCols.length > 0) {
245
          "--connect", NetezzaTestUtils.getNZConnectString()));
135
        assertColMinAndMax(forIdx(0), extraCols[0]);
246
      verifyExport(3, connection);
136
      }
247
    } catch (SQLException sqlE) {
137
    } catch (SQLException sqlE) {
248
      LOG.error("Encountered SQL Exception: " + sqlE);
138
      LOG.error("Encountered SQL Exception: " + sqlE);
249
      sqlE.printStackTrace();
139
      sqlE.printStackTrace();
250
      fail("SQLException when accessing target table. " + sqlE);
140
      fail("SQLException when accessing target table. " + sqlE);
251
    } finally {

   
252
      try {

   
253
        if (null != st) {

   
254
          st.close();

   
255
        }
141
    }
256
      } catch (SQLException sqlE) {

   
257
        LOG.warn("Got SQLException when closing connection: " + sqlE);

   
258
      }
142
  }

    
   
143

   

    
   
144
  /**

    
   
145
   * Test an authenticated export using netezza external table import.

    
   
146
   */

    
   
147
  @Test

    
   
148
  public void testSimpleExport() throws IOException, SQLException {

    
   
149
    String[] argv = getArgv(true, 10, 10);

    
   
150
    runNetezzaTest(getTableName(), argv);
259
    }
151
  }

    
   
152

   

    
   
153
  @Test

    
   
154
  public void testValidExtraArgs() throws Exception {

    
   
155

   

    
   
156
    String [] extraArgs = {

    
   
157
        "--",

    
   
158
        "--log-dir", "/tmp",

    
   
159
        "--max-errors", "2",

    
   
160
     };

    
   
161
    String[] argv = getArgv(true, 10, 10, extraArgs);

    
   
162
    runNetezzaTest(getTableName(), argv);
260
  }
163
  }
261

    
   
164

   
262

    
   
165

   

    
   
166

   
263
  @Override
167
  @Override
264
  public void testMultiMapTextExportWithStaging()
168
  public void testMultiMapTextExportWithStaging() throws IOException,
265
    throws IOException, SQLException {
169
      SQLException {
266
    // disable this test as staging is not supported in direct mode
170
    // disable this test as staging is not supported in direct mode
267
  }
171
  }
268

    
   
172

   
269
  @Override
173
  @Override
270
  public void testMultiTransactionWithStaging()
174
  public void testMultiTransactionWithStaging() throws IOException,
271
    throws IOException, SQLException {
175
      SQLException {
272
    // disable this test as staging is not supported in direct mode
176
    // disable this test as staging is not supported in direct mode
273
  }
177
  }
274

    
   
178

   
275
  @Override
179
  @Override
276
  public void testColumnsExport()
180
  public void testColumnsExport() throws IOException, SQLException {
277
    throws IOException, SQLException {

   
278
    // disable this test as it is not supported in direct mode
181
    // disable this test as it is not supported in direct mode
279
  }
182
  }
280

    
   
183

   
281
  @Override
184
  @Override
282
  public void testSequenceFileExport()
185
  public void testSequenceFileExport() throws IOException, SQLException {
283
    throws IOException, SQLException {

   
284
    // disable this test as it is not supported in direct mode
186
    // disable this test as it is not supported in direct mode
285
  }
187
  }
286
}
188
}
src/test/com/cloudera/sqoop/manager/NetezzaExportManualTest.java
New File
 
src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java
Revision 97399fe New Change
 
src/test/com/cloudera/sqoop/manager/NetezzaTestUtils.java
Revision 9f4c07c New Change
 
  1. src/test/com/cloudera/sqoop/manager/DirectNetezzaExportManualTest.java: Loading...
  2. src/test/com/cloudera/sqoop/manager/NetezzaExportManualTest.java: Loading...
  3. src/test/com/cloudera/sqoop/manager/NetezzaImportManualTest.java: Loading...
  4. src/test/com/cloudera/sqoop/manager/NetezzaTestUtils.java: Loading...