Review Board 1.7.22


SQOOP-749: Exports Using Stored Procedures (Functions)

Review Request #8806 - Created Jan. 3, 2013 and updated

Nick White
SQOOP-749
Reviewers
Sqoop
jarcec
sqoop-trunk
It'd be useful if you could use stored procedures (or functions) to insert data - currently you can only use insert or update statements (or upsert / merges, depending on the SqlManager you're using). This would help sqoop adoption / migration into environments which have existing, SQL-based data import workflows. 

The attached patch adds a --call argument to the export tool that can be used in place of --table.

Note that this patch depends on HSQLDB 2.x, whereas trunk currently depends on 1.8, so it can't be committed until that situation's resolved (see SQOOP-519 and linked tickets). 
I've added a subclass of TestExport to make sure all the usual export functionality works. I've also used it on live data exporting to postgres.
ivy.xml
Revision 1ee60df New Change
[20] 121 lines
[+20]
122
      rev="${commons-logging.version}" conf="common->default"/>
122
      rev="${commons-logging.version}" conf="common->default"/>
123
    <dependency org="log4j" name="log4j" rev="${log4j.version}"
123
    <dependency org="log4j" name="log4j" rev="${log4j.version}"
124
      conf="common->master" />
124
      conf="common->master" />
125
    <dependency org="junit" name="junit" rev="${junit.version}"
125
    <dependency org="junit" name="junit" rev="${junit.version}"
126
      conf="test->default"/>
126
      conf="test->default"/>

    
   
127
    <!-- We're only using H2 for tests as it supports stored

    
   
128
         procedures; once we move to HSQLDB 2.x we can drop

    
   
129
         this -->

    
   
130
    <dependency org="com.h2database" name="h2" rev="${h2.version}"

    
   
131
      conf="test->default"/>
127
    <dependency org="hsqldb" name="hsqldb" rev="${hsqldb.version}"
132
    <dependency org="hsqldb" name="hsqldb" rev="${hsqldb.version}"
128
      conf="common->default;redist->default"/>
133
      conf="common->default;redist->default"/>
129
    <dependency org="commons-io" name="commons-io" rev="${commons-io.version}"
134
    <dependency org="commons-io" name="commons-io" rev="${commons-io.version}"
130
      conf="common->default;redist->default"/>
135
      conf="common->default;redist->default"/>
131
    <dependency org="org.apache.avro" name="avro" rev="${avro.version}"
136
    <dependency org="org.apache.avro" name="avro" rev="${avro.version}"
[+20] [20] 44 lines
ivy/libraries.properties
Revision 4c9e37d New Change
 
src/docs/user/export.txt
Revision 9f600fe New Change
 
src/docs/user/export-purpose.txt
Revision c26eaa7 New Change
 
src/java/org/apache/sqoop/SqoopOptions.java
Revision b0fdfa0 New Change
 
src/java/org/apache/sqoop/manager/ConnManager.java
Revision 115186f New Change
 
src/java/org/apache/sqoop/manager/SqlManager.java
Revision 3a52c6d New Change
 
src/java/org/apache/sqoop/mapreduce/ExportCallOutputFormat.java
New File
 
src/java/org/apache/sqoop/mapreduce/JdbcCallExportJob.java
New File
 
src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java
Revision 00bd910 New Change
 
src/java/org/apache/sqoop/orm/ClassWriter.java
Revision 47e1221 New Change
 
src/java/org/apache/sqoop/tool/BaseSqoopTool.java
Revision b4b2213 New Change
 
src/java/org/apache/sqoop/tool/ExportTool.java
Revision acd296d New Change
 
src/test/com/cloudera/sqoop/SmokeTests.java
Revision 76df6cf New Change
 
src/test/com/cloudera/sqoop/TestConnFactory.java
Revision 893b388 New Change
 
src/test/com/cloudera/sqoop/TestExport.java
Revision eba10aa New Change
 
src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java
Revision be449e4 New Change
 
src/test/com/cloudera/sqoop/testutil/ExportJobTestCase.java
Revision 4f6fd37 New Change
 
src/test/org/apache/sqoop/TestExportUsingProcedure.java
New File
 
  1. ivy.xml: Loading...
  2. ivy/libraries.properties: Loading...
  3. src/docs/user/export.txt: Loading...
  4. src/docs/user/export-purpose.txt: Loading...
  5. src/java/org/apache/sqoop/SqoopOptions.java: Loading...
  6. src/java/org/apache/sqoop/manager/ConnManager.java: Loading...
  7. src/java/org/apache/sqoop/manager/SqlManager.java: Loading...
  8. src/java/org/apache/sqoop/mapreduce/ExportCallOutputFormat.java: Loading...
  9. src/java/org/apache/sqoop/mapreduce/JdbcCallExportJob.java: Loading...
  10. src/java/org/apache/sqoop/mapreduce/JdbcExportJob.java: Loading...
  11. src/java/org/apache/sqoop/orm/ClassWriter.java: Loading...
  12. src/java/org/apache/sqoop/tool/BaseSqoopTool.java: Loading...
  13. src/java/org/apache/sqoop/tool/ExportTool.java: Loading...
  14. src/test/com/cloudera/sqoop/SmokeTests.java: Loading...
  15. src/test/com/cloudera/sqoop/TestConnFactory.java: Loading...
  16. src/test/com/cloudera/sqoop/TestExport.java: Loading...
  17. src/test/com/cloudera/sqoop/manager/PostgresqlExportTest.java: Loading...
  18. src/test/com/cloudera/sqoop/testutil/ExportJobTestCase.java: Loading...
  19. src/test/org/apache/sqoop/TestExportUsingProcedure.java: Loading...